diff options
Diffstat (limited to 'src')
119 files changed, 0 insertions, 29179 deletions
diff --git a/src/boost/assert.hpp b/src/boost/assert.hpp deleted file mode 100644 index 754ebb954b..0000000000 --- a/src/boost/assert.hpp +++ /dev/null @@ -1,38 +0,0 @@ -// -// boost/assert.hpp - BOOST_ASSERT(expr) -// -// Copyright (c) 2001, 2002 Peter Dimov and Multi Media Ltd. -// -// Permission to copy, use, modify, sell and distribute this software -// is granted provided this copyright notice appears in all copies. -// This software is provided "as is" without express or implied -// warranty, and with no claim as to its suitability for any purpose. -// -// Note: There are no include guards. This is intentional. -// -// See http://www.boost.org/libs/utility/assert.html for documentation. -// - -#undef BOOST_ASSERT - -#if defined(BOOST_DISABLE_ASSERTS) - -# define BOOST_ASSERT(expr) ((void)0) - -#elif defined(BOOST_ENABLE_ASSERT_HANDLER) - -#include <boost/current_function.hpp> - -namespace boost -{ - -void assertion_failed(char const * expr, char const * function, char const * file, long line); // user defined - -} // namespace boost - -#define BOOST_ASSERT(expr) ((expr)? ((void)0): ::boost::assertion_failed(#expr, BOOST_CURRENT_FUNCTION, __FILE__, __LINE__)) - -#else -# include <assert.h> -# define BOOST_ASSERT(expr) assert(expr) -#endif diff --git a/src/boost/format.hpp b/src/boost/format.hpp deleted file mode 100644 index f965f0f33e..0000000000 --- a/src/boost/format.hpp +++ /dev/null @@ -1,64 +0,0 @@ -// -*- C++ -*- -// Boost general library 'format' --------------------------- -// See http://www.boost.org for updates, documentation, and revision history. - -// (C) Samuel Krempp 2001 -// krempp@crans.ens-cachan.fr -// Permission to copy, use, modify, sell and -// distribute this software is granted provided this copyright notice appears -// in all copies. This software is provided "as is" without express or implied -// warranty, and with no claim as to its suitability for any purpose. - -// ideas taken from Rüdiger Loos's format class -// and Karl Nelson's ofstream - -// ---------------------------------------------------------------------------- -// format.hpp : primary header -// ---------------------------------------------------------------------------- - -#ifndef BOOST_FORMAT_HPP -#define BOOST_FORMAT_HPP - -#include <vector> -#include <string> -#include <sstream> -#include <cassert> - -#if HAVE_LOCALE -#include <locale> -#else -#define BOOST_NO_STD_LOCALE -#define BOOST_NO_LOCALE_ISIDIGIT -#include <cctype> -#endif - -#include <boost/format/macros_default.hpp> - - -// **** Forward declarations ---------------------------------- -#include <boost/format/format_fwd.hpp> // basic_format<Ch,Tr>, and other frontends -#include <boost/format/internals_fwd.hpp> // misc forward declarations for internal use - - -// **** Auxiliary structs (stream_format_state<Ch,Tr> , and format_item<Ch,Tr> ) -#include <boost/format/internals.hpp> - -// **** Format class interface -------------------------------- -#include <boost/format/format_class.hpp> - -// **** Exceptions ----------------------------------------------- -#include <boost/format/exceptions.hpp> - -// **** Implementation ------------------------------------------- -//#include <boost/format/format_implementation.hpp> // member functions - -#include <boost/format/group.hpp> // class for grouping arguments - -#include <boost/format/feed_args.hpp> // argument-feeding functions -//#include <boost/format/parsing.hpp> // format-string parsing (member-)functions - -// **** Implementation of the free functions ---------------------- -//#include <boost/format/free_funcs.hpp> - - -#endif // BOOST_FORMAT_HPP diff --git a/src/boost/format/exceptions.hpp b/src/boost/format/exceptions.hpp deleted file mode 100644 index 79e452449e..0000000000 --- a/src/boost/format/exceptions.hpp +++ /dev/null @@ -1,96 +0,0 @@ -// -*- C++ -*- -// Boost general library 'format' --------------------------- -// See http://www.boost.org for updates, documentation, and revision history. - -// (C) Samuel Krempp 2001 -// krempp@crans.ens-cachan.fr -// Permission to copy, use, modify, sell and -// distribute this software is granted provided this copyright notice appears -// in all copies. This software is provided "as is" without express or implied -// warranty, and with no claim as to its suitability for any purpose. - -// ideas taken from Rüdiger Loos's format class -// and Karl Nelson's ofstream (also took its parsing code as basis for printf parsing) - -// ------------------------------------------------------------------------------ -// exceptions.hpp -// ------------------------------------------------------------------------------ - - -#ifndef BOOST_FORMAT_EXCEPTIONS_HPP -#define BOOST_FORMAT_EXCEPTIONS_HPP - - -#include <stdexcept> - - -namespace boost { - -namespace io { - -// **** exceptions ----------------------------------------------- - -class format_error : public std::exception -{ -public: - format_error() {} - virtual const char *what() const throw() - { - return "boost::format_error: " - "format generic failure"; - } -}; - -class bad_format_string : public format_error -{ -public: - bad_format_string() {} - virtual const char *what() const throw() - { - return "boost::bad_format_string: " - "format-string is ill-formed"; - } -}; - -class too_few_args : public format_error -{ -public: - too_few_args() {} - virtual const char *what() const throw() - { - return "boost::too_few_args: " - "format-string refered to more arguments than were passed"; - } -}; - -class too_many_args : public format_error -{ -public: - too_many_args() {} - virtual const char *what() const throw() - { - return "boost::too_many_args: " - "format-string refered to less arguments than were passed"; - } -}; - - -class out_of_range : public format_error -{ -public: - out_of_range() {} - virtual const char *what() const throw() - { - return "boost::out_of_range: " - "tried to refer to an argument (or item) number which is out of range, " - "according to the format string."; - } -}; - - -} // namespace io - -} // namespace boost - - -#endif // BOOST_FORMAT_EXCEPTIONS_HPP diff --git a/src/boost/format/feed_args.hpp b/src/boost/format/feed_args.hpp deleted file mode 100644 index 3d0b47b4a1..0000000000 --- a/src/boost/format/feed_args.hpp +++ /dev/null @@ -1,247 +0,0 @@ -// -*- C++ -*- -// Boost general library 'format' --------------------------- -// See http://www.boost.org for updates, documentation, and revision history. - -// (C) Samuel Krempp 2001 -// krempp@crans.ens-cachan.fr -// Permission to copy, use, modify, sell and -// distribute this software is granted provided this copyright notice appears -// in all copies. This software is provided "as is" without express or implied -// warranty, and with no claim as to its suitability for any purpose. - -// ideas taken from Rüdiger Loos's format class -// and Karl Nelson's ofstream - -// ---------------------------------------------------------------------------- -// feed_args.hpp : functions for processing each argument -// (feed, feed_manip, and distribute) -// ---------------------------------------------------------------------------- - - -#ifndef BOOST_FORMAT_FEED_ARGS_HPP -#define BOOST_FORMAT_FEED_ARGS_HPP - -#include "boost/format/format_class.hpp" -#include "boost/format/group.hpp" - -#include "boost/throw_exception.hpp" - -namespace boost { -namespace io { -namespace detail { -namespace { - - inline - void empty_buf(BOOST_IO_STD ostringstream & os) { - static const std::string emptyStr; - os.str(emptyStr); - } - - void do_pad( std::string & s, - std::streamsize w, - const char c, - std::ios::fmtflags f, - bool center) - // applies centered / left / right padding to the string s. - // Effects : string s is padded. - { - std::streamsize n=w-s.size(); - if(n<=0) { - return; - } - if(center) - { - s.reserve(w); // allocate once for the 2 inserts - const std::streamsize n1 = n /2, n0 = n - n1; - s.insert(s.begin(), n0, c); - s.append(n1, c); - } - else - { - if(f & std::ios::left) { - s.append(n, c); - } - else { - s.insert(s.begin(), n, c); - } - } - } // -do_pad(..) - - - template<class T> inline - void put_head(BOOST_IO_STD ostream& , const T& ) { - } - - template<class T> inline - void put_head( BOOST_IO_STD ostream& os, const group1<T>& x ) { - os << group_head(x.a1_); // send the first N-1 items, not the last - } - - template<class T> inline - void put_last( BOOST_IO_STD ostream& os, const T& x ) { - os << x ; - } - - template<class T> inline - void put_last( BOOST_IO_STD ostream& os, const group1<T>& x ) { - os << group_last(x.a1_); // this selects the last element - } - -#ifndef BOOST_NO_OVERLOAD_FOR_NON_CONST - template<class T> inline - void put_head( BOOST_IO_STD ostream& , T& ) { - } - - template<class T> inline - void put_last( BOOST_IO_STD ostream& os, T& x ) { - os << x ; - } -#endif - - - - -template<class T> -void put( T x, - const format_item& specs, - std::string & res, - BOOST_IO_STD ostringstream& oss_ ) -{ - // does the actual conversion of x, with given params, into a string - // using the *supplied* strinstream. (the stream state is important) - - typedef std::string string_t; - typedef format_item format_item_t; - - stream_format_state prev_state(oss_); - - specs.state_.apply_on(oss_); - - // in case x is a group, apply the manip part of it, - // in order to find width - put_head( oss_, x ); - empty_buf( oss_); - - const std::streamsize w=oss_.width(); - const std::ios::fmtflags fl=oss_.flags(); - const bool internal = (fl & std::ios::internal) != 0; - const bool two_stepped_padding = internal - && ! ( specs.pad_scheme_ & format_item_t::spacepad ) - && specs.truncate_ < 0 ; - - - if(! two_stepped_padding) - { - if(w>0) // handle simple padding via do_pad, not natively in stream - oss_.width(0); - put_last( oss_, x); - res = oss_.str(); - - if (specs.truncate_ >= 0) - res.erase(specs.truncate_); - - // complex pads : - if(specs.pad_scheme_ & format_item_t::spacepad) - { - if( res.size()==0 || ( res[0]!='+' && res[0]!='-' )) - { - res.insert(res.begin(), 1, ' '); // insert 1 space at pos 0 - } - } - if(w > 0) // need do_pad - { - do_pad(res,w,oss_.fill(), fl, (specs.pad_scheme_ & format_item_t::centered) !=0 ); - } - } - else // 2-stepped padding - { - put_last( oss_, x); // oss_.width() may result in padding. - res = oss_.str(); - - if (specs.truncate_ >= 0) - res.erase(specs.truncate_); - - if( res.size() - w > 0) - { // length w exceeded - // either it was multi-output with first output padding up all width.. - // either it was one big arg and we are fine. - empty_buf( oss_); - oss_.width(0); - put_last(oss_, x ); - string_t tmp = oss_.str(); // minimal-length output - std::streamsize d; - if( (d=w - tmp.size()) <=0 ) - { - // minimal length is already >= w, so no padding (cool!) - res.swap(tmp); - } - else - { // hum.. we need to pad (it was necessarily multi-output) - typedef typename string_t::size_type size_type; - size_type i = 0; - while( i<tmp.size() && tmp[i] == res[i] ) // find where we should pad. - ++i; - tmp.insert(i, static_cast<size_type>( d ), oss_.fill()); - res.swap( tmp ); - } - } - else - { // okay, only one thing was printed and padded, so res is fine. - } - } - - prev_state.apply_on(oss_); - empty_buf( oss_); - oss_.clear(); -} // end- put(..) - - -} // local namespace - - - - - -template<class T> -void distribute(basic_format& self, T x) - // call put(x, ..) on every occurence of the current argument : -{ - if(self.cur_arg_ >= self.num_args_) - { - if( self.exceptions() & too_many_args_bit ) - boost::throw_exception(too_many_args()); // too many variables have been supplied ! - else return; - } - for(unsigned long i=0; i < self.items_.size(); ++i) - { - if(self.items_[i].argN_ == self.cur_arg_) - { - put<T> (x, self.items_[i], self.items_[i].res_, self.oss_ ); - } - } -} - -template<class T> -basic_format& feed(basic_format& self, T x) -{ - if(self.dumped_) self.clear(); - distribute<T> (self, x); - ++self.cur_arg_; - if(self.bound_.size() != 0) - { - while( self.cur_arg_ < self.num_args_ && self.bound_[self.cur_arg_] ) - ++self.cur_arg_; - } - - // this arg is finished, reset the stream's format state - self.state0_.apply_on(self.oss_); - return self; -} - - -} // namespace detail -} // namespace io -} // namespace boost - - -#endif // BOOST_FORMAT_FEED_ARGS_HPP diff --git a/src/boost/format/format_class.hpp b/src/boost/format/format_class.hpp deleted file mode 100644 index 6875623acb..0000000000 --- a/src/boost/format/format_class.hpp +++ /dev/null @@ -1,135 +0,0 @@ -// -*- C++ -*- -// Boost general library 'format' --------------------------- -// See http://www.boost.org for updates, documentation, and revision history. - -// (C) Samuel Krempp 2001 -// krempp@crans.ens-cachan.fr -// Permission to copy, use, modify, sell and -// distribute this software is granted provided this copyright notice appears -// in all copies. This software is provided "as is" without express or implied -// warranty, and with no claim as to its suitability for any purpose. - -// ideas taken from Rüdiger Loos's format class -// and Karl Nelson's ofstream (also took its parsing code as basis for printf parsing) - -// ------------------------------------------------------------------------------ -// format_class.hpp : class interface -// ------------------------------------------------------------------------------ - - -#ifndef BOOST_FORMAT_CLASS_HPP -#define BOOST_FORMAT_CLASS_HPP - -#include <vector> -#include <string> - -#include <boost/format/format_fwd.hpp> -#include <boost/format/internals_fwd.hpp> - -#include <boost/format/internals.hpp> - -namespace boost { - -class basic_format -{ -public: - typedef std::string string_t; - typedef BOOST_IO_STD ostringstream internal_stream_t; -private: - typedef BOOST_IO_STD ostream stream_t; - typedef io::detail::stream_format_state stream_format_state; - typedef io::detail::format_item format_item_t; - -public: - basic_format(const char* str); - basic_format(const string_t& s); -#ifndef BOOST_NO_STD_LOCALE - basic_format(const char* str, const std::locale & loc); - basic_format(const string_t& s, const std::locale & loc); -#endif // no locale - basic_format(const basic_format& x); - basic_format& operator= (const basic_format& x); - - basic_format& clear(); // empty the string buffers (except bound arguments, see clear_binds() ) - - // pass arguments through those operators : - template<class T> basic_format& operator%(const T& x) - { - return io::detail::feed<const T&>(*this,x); - } - -#ifndef BOOST_NO_OVERLOAD_FOR_NON_CONST - template<class T> basic_format& operator%(T& x) - { - return io::detail::feed<T&>(*this,x); - } -#endif - - - // system for binding arguments : - template<class T> - basic_format& bind_arg(int argN, const T& val) - { - return io::detail::bind_arg_body(*this, argN, val); - } - basic_format& clear_bind(int argN); - basic_format& clear_binds(); - - // modify the params of a directive, by applying a manipulator : - template<class T> - basic_format& modify_item(int itemN, const T& manipulator) - { - return io::detail::modify_item_body(*this, itemN, manipulator) ; - } - - // Choosing which errors will throw exceptions : - unsigned char exceptions() const; - unsigned char exceptions(unsigned char newexcept); - - // final output - string_t str() const; - friend BOOST_IO_STD ostream& - operator<< ( BOOST_IO_STD ostream& , const basic_format& ); - - - template<class T> friend basic_format& - io::detail::feed(basic_format&, T); - - template<class T> friend - void io::detail::distribute(basic_format&, T); - - template<class T> friend - basic_format& io::detail::modify_item_body(basic_format&, int, const T&); - - template<class T> friend - basic_format& io::detail::bind_arg_body(basic_format&, int, const T&); - -// make the members private only if the friend templates are supported -private: - - // flag bits, used for style_ - enum style_values { ordered = 1, // set only if all directives are positional directives - special_needs = 4 }; - - // parse the format string : - void parse(const string_t&); - - int style_; // style of format-string : positional or not, etc - int cur_arg_; // keep track of wich argument will come - int num_args_; // number of expected arguments - mutable bool dumped_; // true only after call to str() or << - std::vector<format_item_t> items_; // vector of directives (aka items) - string_t prefix_; // piece of string to insert before first item - - std::vector<bool> bound_; // stores which arguments were bound - // size = num_args OR zero - internal_stream_t oss_; // the internal stream. - stream_format_state state0_; // reference state for oss_ - unsigned char exceptions_; -}; // class basic_format - - -} // namespace boost - - -#endif // BOOST_FORMAT_CLASS_HPP diff --git a/src/boost/format/format_fwd.hpp b/src/boost/format/format_fwd.hpp deleted file mode 100644 index 97c55f6684..0000000000 --- a/src/boost/format/format_fwd.hpp +++ /dev/null @@ -1,49 +0,0 @@ -// -*- C++ -*- -// Boost general library 'format' --------------------------- -// See http://www.boost.org for updates, documentation, and revision history. - -// (C) Samuel Krempp 2001 -// krempp@crans.ens-cachan.fr -// Permission to copy, use, modify, sell and -// distribute this software is granted provided this copyright notice appears -// in all copies. This software is provided "as is" without express or implied -// warranty, and with no claim as to its suitability for any purpose. - -// ideas taken from Rüdiger Loos's format class -// and Karl Nelson's ofstream (also took its parsing code as basis for printf parsing) - -// ------------------------------------------------------------------------------ -// format_fwd.hpp : forward declarations, for primary header format.hpp -// ------------------------------------------------------------------------------ - -#ifndef BOOST_FORMAT_FWD_HPP -#define BOOST_FORMAT_FWD_HPP - -#include <string> -#include <iosfwd> - -namespace boost { - -class basic_format; - -typedef basic_format format; - -namespace io { -enum format_error_bits { bad_format_string_bit = 1, - too_few_args_bit = 2, too_many_args_bit = 4, - out_of_range_bit = 8, - all_error_bits = 255, no_error_bits=0 }; - -// Convertion: format to string -std::string str(const basic_format& ) ; - -} // namespace io - - -BOOST_IO_STD ostream& -operator<<( BOOST_IO_STD ostream&, const basic_format&); - - -} // namespace boost - -#endif // BOOST_FORMAT_FWD_HPP diff --git a/src/boost/format/format_implementation.cc b/src/boost/format/format_implementation.cc deleted file mode 100644 index aa191afe11..0000000000 --- a/src/boost/format/format_implementation.cc +++ /dev/null @@ -1,256 +0,0 @@ -// -*- C++ -*- -// Boost general library format --------------------------- -// See http://www.boost.org for updates, documentation, and revision history. - -// (C) Samuel Krempp 2001 -// krempp@crans.ens-cachan.fr -// Permission to copy, use, modify, sell and -// distribute this software is granted provided this copyright notice appears -// in all copies. This software is provided "as is" without express or implied -// warranty, and with no claim as to its suitability for any purpose. - -// ideas taken from Rüdiger Loos's format class -// and Karl Nelson's ofstream - -// ---------------------------------------------------------------------------- -// format_implementation.hpp Implementation of the basic_format class -// ---------------------------------------------------------------------------- - - -#ifndef BOOST_FORMAT_IMPLEMENTATION_HPP -#define BOOST_FORMAT_IMPLEMENTATION_HPP - -#include <boost/throw_exception.hpp> -#include <boost/assert.hpp> -#include <boost/format.hpp> - -namespace boost { - -// -------- format:: ------------------------------------------- -basic_format::basic_format(const char* str) - : style_(0), cur_arg_(0), num_args_(0), dumped_(false), - items_(), oss_(), exceptions_(io::all_error_bits) -{ - state0_.set_by_stream(oss_); - string_t emptyStr; - if( !str) str = emptyStr.c_str(); - parse( str ); -} - -#ifndef BOOST_NO_STD_LOCALE -basic_format::basic_format(const char* str, const std::locale & loc) - : style_(0), cur_arg_(0), num_args_(0), dumped_(false), - items_(), oss_(), exceptions_(io::all_error_bits) -{ - oss_.imbue( loc ); - state0_.set_by_stream(oss_); - string_t emptyStr; - if( !str) str = emptyStr.c_str(); - parse( str ); -} - -basic_format::basic_format(const string_t& s, const std::locale & loc) - : style_(0), cur_arg_(0), num_args_(0), dumped_(false), - items_(), oss_(), exceptions_(io::all_error_bits) -{ - oss_.imbue( loc ); - state0_.set_by_stream(oss_); - parse(s); -} -#endif //BOOST_NO_STD_LOCALE - -basic_format::basic_format(const string_t& s) - : style_(0), cur_arg_(0), num_args_(0), dumped_(false), - items_(), oss_(), exceptions_(io::all_error_bits) -{ - state0_.set_by_stream(oss_); - parse(s); -} - -basic_format:: basic_format(const basic_format& x) - : style_(x.style_), cur_arg_(x.cur_arg_), num_args_(x.num_args_), dumped_(false), - items_(x.items_), prefix_(x.prefix_), bound_(x.bound_), - oss_(), // <- we obviously can't copy x.oss_ - state0_(x.state0_), exceptions_(x.exceptions_) -{ - state0_.apply_on(oss_); -} - -basic_format& basic_format::operator= (const basic_format& x) -{ - if(this == &x) - return *this; - state0_ = x.state0_; - state0_.apply_on(oss_); - - // plus all the other (trivial) assignments : - exceptions_ = x.exceptions_; - items_ = x.items_; - prefix_ = x.prefix_; - bound_=x.bound_; - style_=x.style_; - cur_arg_=x.cur_arg_; - num_args_=x.num_args_; - dumped_=x.dumped_; - return *this; -} - - -unsigned char basic_format::exceptions() const -{ - return exceptions_; -} - -unsigned char basic_format::exceptions(unsigned char newexcept) -{ - unsigned char swp = exceptions_; - exceptions_ = newexcept; - return swp; -} - - -basic_format& basic_format ::clear() - // empty the string buffers (except bound arguments, see clear_binds() ) - // and make the format object ready for formatting a new set of arguments -{ - BOOST_ASSERT( bound_.size()==0 || num_args_ == static_cast<int>(bound_.size()) ); - - for(unsigned long i=0; i<items_.size(); ++i){ - items_[i].state_ = items_[i].ref_state_; - // clear converted strings only if the corresponding argument is not bound : - if( bound_.size()==0 || !bound_[ items_[i].argN_ ] ) items_[i].res_.resize(0); - } - cur_arg_=0; dumped_=false; - // maybe first arg is bound: - if(bound_.size() != 0) - { - while(cur_arg_ < num_args_ && bound_[cur_arg_] ) ++cur_arg_; - } - return *this; -} - -basic_format& basic_format ::clear_binds() - // cancel all bindings, and clear() -{ - bound_.resize(0); - clear(); - return *this; -} - -basic_format& basic_format::clear_bind(int argN) - // cancel the binding of ONE argument, and clear() -{ - if(argN<1 || argN > num_args_ || bound_.size()==0 || !bound_[argN-1] ) - { - if( exceptions() & io::out_of_range_bit ) - boost::throw_exception(io::out_of_range()); // arg not in range. - else return *this; - } - bound_[argN-1]=false; - clear(); - return *this; -} - - - -std::string basic_format::str() const -{ - dumped_=true; - if(items_.size()==0) - return prefix_; - if( cur_arg_ < num_args_) - if( exceptions() & io::too_few_args_bit ) - boost::throw_exception(io::too_few_args()); // not enough variables have been supplied ! - - unsigned long sz = prefix_.size(); - unsigned long i; - for(i=0; i < items_.size(); ++i) - sz += items_[i].res_.size() + items_[i].appendix_.size(); - string_t res; - res.reserve(sz); - - res += prefix_; - for(i=0; i < items_.size(); ++i) - { - const format_item_t& item = items_[i]; - res += item.res_; - if( item.argN_ == format_item_t::argN_tabulation) - { - BOOST_ASSERT( item.pad_scheme_ & format_item_t::tabulation); - std::streamsize n = item.state_.width_ - res.size(); - if( n > 0 ) - res.append( n, item.state_.fill_ ); - } - res += item.appendix_; - } - return res; -} - -namespace io { -namespace detail { - -template<class T> -basic_format& bind_arg_body( basic_format& self, - int argN, - const T& val) - // bind one argument to a fixed value - // this is persistent over clear() calls, thus also over str() and << -{ - if(self.dumped_) self.clear(); // needed, because we will modify cur_arg_.. - if(argN<1 || argN > self.num_args_) - { - if( self.exceptions() & io::out_of_range_bit ) - boost::throw_exception(io::out_of_range()); // arg not in range. - else return self; - } - if(self.bound_.size()==0) - self.bound_.assign(self.num_args_,false); - else - BOOST_ASSERT( self.num_args_ == static_cast<signed int>(self.bound_.size()) ); - int o_cur_arg = self.cur_arg_; - self.cur_arg_ = argN-1; // arrays begin at 0 - - self.bound_[self.cur_arg_]=false; // if already set, we unset and re-sets.. - self.operator%(val); // put val at the right place, because cur_arg is set - - - // Now re-position cur_arg before leaving : - self.cur_arg_ = o_cur_arg; - self.bound_[argN-1]=true; - if(self.cur_arg_ == argN-1 ) - // hum, now this arg is bound, so move to next free arg - { - while(self.cur_arg_ < self.num_args_ && self.bound_[self.cur_arg_]) ++self.cur_arg_; - } - // In any case, we either have all args, or are on a non-binded arg : - BOOST_ASSERT( self.cur_arg_ >= self.num_args_ || ! self.bound_[self.cur_arg_]); - return self; -} - -template<class T> -basic_format& modify_item_body( basic_format& self, - int itemN, - const T& manipulator) - // applies a manipulator to the format_item describing a given directive. - // this is a permanent change, clear or clear_binds won't cancel that. -{ - if(itemN<1 || itemN >= static_cast<signed int>(self.items_.size() )) - { - if( self.exceptions() & io::out_of_range_bit ) - boost::throw_exception(io::out_of_range()); // item not in range. - else return self; - } - self.items_[itemN-1].ref_state_.apply_manip( manipulator ); - self.items_[itemN-1].state_ = self.items_[itemN-1].ref_state_; - return self; -} - -} // namespace detail - -} // namespace io - -} // namespace boost - - - -#endif // BOOST_FORMAT_IMPLEMENTATION_HPP diff --git a/src/boost/format/free_funcs.cc b/src/boost/format/free_funcs.cc deleted file mode 100644 index 151db37a0a..0000000000 --- a/src/boost/format/free_funcs.cc +++ /dev/null @@ -1,71 +0,0 @@ -// -*- C++ -*- -// Boost general library 'format' --------------------------- -// See http://www.boost.org for updates, documentation, and revision history. - -// (C) Samuel Krempp 2001 -// krempp@crans.ens-cachan.fr -// Permission to copy, use, modify, sell and -// distribute this software is granted provided this copyright notice appears -// in all copies. This software is provided "as is" without express or implied -// warranty, and with no claim as to its suitability for any purpose. - -// ideas taken from Rüdiger Loos's format class -// and Karl Nelson's ofstream (also took its parsing code as basis for printf parsing) - -// ------------------------------------------------------------------------------ -// free_funcs.hpp : implementation of the free functions declared in namespace format -// ------------------------------------------------------------------------------ - -#ifndef BOOST_FORMAT_FUNCS_HPP -#define BOOST_FORMAT_FUNCS_HPP - -#include "boost/format.hpp" -#include "boost/throw_exception.hpp" - -namespace boost { - -namespace io { - inline - std::string str(const basic_format& f) - // adds up all pieces of strings and converted items, and return the formatted string - { - return f.str(); - } -} // - namespace io - -BOOST_IO_STD ostream& -operator<<( BOOST_IO_STD ostream& os, - const boost::basic_format& f) - // effect: "return os << str(f);" but we can try to do it faster -{ - typedef boost::basic_format format_t; - if(f.items_.size()==0) - os << f.prefix_; - else { - if(f.cur_arg_ < f.num_args_) - if( f.exceptions() & io::too_few_args_bit ) - boost::throw_exception(io::too_few_args()); // not enough variables have been supplied ! - if(f.style_ & format_t::special_needs) - os << f.str(); - else { - // else we dont have to count chars output, so we dump directly to os : - os << f.prefix_; - for(unsigned long i=0; i<f.items_.size(); ++i) - { - const format_t::format_item_t& item = f.items_[i]; - os << item.res_; - os << item.appendix_; - - } - } - } - f.dumped_=true; - return os; -} - - - -} // namespace boost - - -#endif // BOOST_FORMAT_FUNCS_HPP diff --git a/src/boost/format/group.hpp b/src/boost/format/group.hpp deleted file mode 100644 index ac63f3f0ba..0000000000 --- a/src/boost/format/group.hpp +++ /dev/null @@ -1,680 +0,0 @@ - -// -*- C++ -*- -// Boost general library 'format' --------------------------- -// See http://www.boost.org for updates, documentation, and revision history. - -// (C) Samuel Krempp 2001 -// krempp@crans.ens-cachan.fr -// Permission to copy, use, modify, sell and -// distribute this software is granted provided this copyright notice appears -// in all copies. This software is provided "as is" without express or implied -// warranty, and with no claim as to its suitability for any purpose. - -// ideas taken from Rüdiger Loos's format class -// and Karl Nelson's ofstream - -// ---------------------------------------------------------------------------- - -// group.hpp : encapsulates a group of manipulators along with an argument -// -// group_head : cut the last element of a group out. -// (is overloaded below on each type of group) - -// group_last : returns the last element of a group -// (is overloaded below on each type of group) - -// ---------------------------------------------------------------------------- - - -#ifndef BOOST_FORMAT_GROUP_HPP -#define BOOST_FORMAT_GROUP_HPP - - -namespace boost { -namespace io { - - -namespace detail { - - -// empty group, but useful even though. -struct group0 -{ - group0() {} -}; - -template <class Ch, class Tr> -inline -BOOST_IO_STD ostream& -operator << ( BOOST_IO_STD ostream& os, - const group0& ) -{ - return os; -} - -template <class T1> -struct group1 -{ - T1 a1_; - group1(T1 a1) - : a1_(a1) - {} -}; - -template <class Ch, class Tr, class T1> -inline -BOOST_IO_STD ostream& -operator << (BOOST_IO_STD ostream& os, - const group1<T1>& x) -{ - os << x.a1_; - return os; -} - - - - -template <class T1,class T2> -struct group2 -{ - T1 a1_; - T2 a2_; - group2(T1 a1,T2 a2) - : a1_(a1),a2_(a2) - {} -}; - -template <class Ch, class Tr, class T1,class T2> -inline -BOOST_IO_STD ostream& -operator << (BOOST_IO_STD ostream& os, - const group2<T1,T2>& x) -{ - os << x.a1_<< x.a2_; - return os; -} - -template <class T1,class T2,class T3> -struct group3 -{ - T1 a1_; - T2 a2_; - T3 a3_; - group3(T1 a1,T2 a2,T3 a3) - : a1_(a1),a2_(a2),a3_(a3) - {} -}; - -template <class Ch, class Tr, class T1,class T2,class T3> -inline -BOOST_IO_STD ostream& -operator << (BOOST_IO_STD ostream& os, - const group3<T1,T2,T3>& x) -{ - os << x.a1_<< x.a2_<< x.a3_; - return os; -} - -template <class T1,class T2,class T3,class T4> -struct group4 -{ - T1 a1_; - T2 a2_; - T3 a3_; - T4 a4_; - group4(T1 a1,T2 a2,T3 a3,T4 a4) - : a1_(a1),a2_(a2),a3_(a3),a4_(a4) - {} -}; - -template <class Ch, class Tr, class T1,class T2,class T3,class T4> -inline -BOOST_IO_STD ostream& -operator << (BOOST_IO_STD ostream& os, - const group4<T1,T2,T3,T4>& x) -{ - os << x.a1_<< x.a2_<< x.a3_<< x.a4_; - return os; -} - -template <class T1,class T2,class T3,class T4,class T5> -struct group5 -{ - T1 a1_; - T2 a2_; - T3 a3_; - T4 a4_; - T5 a5_; - group5(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5) - : a1_(a1),a2_(a2),a3_(a3),a4_(a4),a5_(a5) - {} -}; - -template <class Ch, class Tr, class T1,class T2,class T3,class T4,class T5> -inline -BOOST_IO_STD ostream& -operator << (BOOST_IO_STD ostream& os, - const group5<T1,T2,T3,T4,T5>& x) -{ - os << x.a1_<< x.a2_<< x.a3_<< x.a4_<< x.a5_; - return os; -} - -template <class T1,class T2,class T3,class T4,class T5,class T6> -struct group6 -{ - T1 a1_; - T2 a2_; - T3 a3_; - T4 a4_; - T5 a5_; - T6 a6_; - group6(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6) - : a1_(a1),a2_(a2),a3_(a3),a4_(a4),a5_(a5),a6_(a6) - {} -}; - -template <class Ch, class Tr, class T1,class T2,class T3,class T4,class T5,class T6> -inline -BOOST_IO_STD ostream& -operator << (BOOST_IO_STD ostream& os, - const group6<T1,T2,T3,T4,T5,T6>& x) -{ - os << x.a1_<< x.a2_<< x.a3_<< x.a4_<< x.a5_<< x.a6_; - return os; -} - -template <class T1,class T2,class T3,class T4,class T5,class T6,class T7> -struct group7 -{ - T1 a1_; - T2 a2_; - T3 a3_; - T4 a4_; - T5 a5_; - T6 a6_; - T7 a7_; - group7(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7) - : a1_(a1),a2_(a2),a3_(a3),a4_(a4),a5_(a5),a6_(a6),a7_(a7) - {} -}; - -template <class Ch, class Tr, class T1,class T2,class T3,class T4,class T5,class T6,class T7> -inline -BOOST_IO_STD ostream& -operator << (BOOST_IO_STD ostream& os, - const group7<T1,T2,T3,T4,T5,T6,T7>& x) -{ - os << x.a1_<< x.a2_<< x.a3_<< x.a4_<< x.a5_<< x.a6_<< x.a7_; - return os; -} - -template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8> -struct group8 -{ - T1 a1_; - T2 a2_; - T3 a3_; - T4 a4_; - T5 a5_; - T6 a6_; - T7 a7_; - T8 a8_; - group8(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7,T8 a8) - : a1_(a1),a2_(a2),a3_(a3),a4_(a4),a5_(a5),a6_(a6),a7_(a7),a8_(a8) - {} -}; - -template <class Ch, class Tr, class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8> -inline -BOOST_IO_STD ostream& -operator << (BOOST_IO_STD ostream& os, - const group8<T1,T2,T3,T4,T5,T6,T7,T8>& x) -{ - os << x.a1_<< x.a2_<< x.a3_<< x.a4_<< x.a5_<< x.a6_<< x.a7_<< x.a8_; - return os; -} - -template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9> -struct group9 -{ - T1 a1_; - T2 a2_; - T3 a3_; - T4 a4_; - T5 a5_; - T6 a6_; - T7 a7_; - T8 a8_; - T9 a9_; - group9(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7,T8 a8,T9 a9) - : a1_(a1),a2_(a2),a3_(a3),a4_(a4),a5_(a5),a6_(a6),a7_(a7),a8_(a8),a9_(a9) - {} -}; - -template <class Ch, class Tr, class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9> -inline -BOOST_IO_STD ostream& -operator << (BOOST_IO_STD ostream& os, - const group9<T1,T2,T3,T4,T5,T6,T7,T8,T9>& x) -{ - os << x.a1_<< x.a2_<< x.a3_<< x.a4_<< x.a5_<< x.a6_<< x.a7_<< x.a8_<< x.a9_; - return os; -} - -template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9,class T10> -struct group10 -{ - T1 a1_; - T2 a2_; - T3 a3_; - T4 a4_; - T5 a5_; - T6 a6_; - T7 a7_; - T8 a8_; - T9 a9_; - T10 a10_; - group10(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7,T8 a8,T9 a9,T10 a10) - : a1_(a1),a2_(a2),a3_(a3),a4_(a4),a5_(a5),a6_(a6),a7_(a7),a8_(a8),a9_(a9),a10_(a10) - {} -}; - -template <class Ch, class Tr, class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9,class T10> -inline -BOOST_IO_STD ostream& -operator << (BOOST_IO_STD ostream& os, - const group10<T1,T2,T3,T4,T5,T6,T7,T8,T9,T10>& x) -{ - os << x.a1_<< x.a2_<< x.a3_<< x.a4_<< x.a5_<< x.a6_<< x.a7_<< x.a8_<< x.a9_<< x.a10_; - return os; -} - - - - -template <class T1,class T2> -inline -group1<T1> -group_head( group2<T1,T2> const& x) -{ - return group1<T1> (x.a1_); -} - -template <class T1,class T2> -inline -group1<T2> -group_last( group2<T1,T2> const& x) -{ - return group1<T2> (x.a2_); -} - - - -template <class T1,class T2,class T3> -inline -group2<T1,T2> -group_head( group3<T1,T2,T3> const& x) -{ - return group2<T1,T2> (x.a1_,x.a2_); -} - -template <class T1,class T2,class T3> -inline -group1<T3> -group_last( group3<T1,T2,T3> const& x) -{ - return group1<T3> (x.a3_); -} - - - -template <class T1,class T2,class T3,class T4> -inline -group3<T1,T2,T3> -group_head( group4<T1,T2,T3,T4> const& x) -{ - return group3<T1,T2,T3> (x.a1_,x.a2_,x.a3_); -} - -template <class T1,class T2,class T3,class T4> -inline -group1<T4> -group_last( group4<T1,T2,T3,T4> const& x) -{ - return group1<T4> (x.a4_); -} - - - -template <class T1,class T2,class T3,class T4,class T5> -inline -group4<T1,T2,T3,T4> -group_head( group5<T1,T2,T3,T4,T5> const& x) -{ - return group4<T1,T2,T3,T4> (x.a1_,x.a2_,x.a3_,x.a4_); -} - -template <class T1,class T2,class T3,class T4,class T5> -inline -group1<T5> -group_last( group5<T1,T2,T3,T4,T5> const& x) -{ - return group1<T5> (x.a5_); -} - - - -template <class T1,class T2,class T3,class T4,class T5,class T6> -inline -group5<T1,T2,T3,T4,T5> -group_head( group6<T1,T2,T3,T4,T5,T6> const& x) -{ - return group5<T1,T2,T3,T4,T5> (x.a1_,x.a2_,x.a3_,x.a4_,x.a5_); -} - -template <class T1,class T2,class T3,class T4,class T5,class T6> -inline -group1<T6> -group_last( group6<T1,T2,T3,T4,T5,T6> const& x) -{ - return group1<T6> (x.a6_); -} - - - -template <class T1,class T2,class T3,class T4,class T5,class T6,class T7> -inline -group6<T1,T2,T3,T4,T5,T6> -group_head( group7<T1,T2,T3,T4,T5,T6,T7> const& x) -{ - return group6<T1,T2,T3,T4,T5,T6> (x.a1_,x.a2_,x.a3_,x.a4_,x.a5_,x.a6_); -} - -template <class T1,class T2,class T3,class T4,class T5,class T6,class T7> -inline -group1<T7> -group_last( group7<T1,T2,T3,T4,T5,T6,T7> const& x) -{ - return group1<T7> (x.a7_); -} - - - -template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8> -inline -group7<T1,T2,T3,T4,T5,T6,T7> -group_head( group8<T1,T2,T3,T4,T5,T6,T7,T8> const& x) -{ - return group7<T1,T2,T3,T4,T5,T6,T7> (x.a1_,x.a2_,x.a3_,x.a4_,x.a5_,x.a6_,x.a7_); -} - -template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8> -inline -group1<T8> -group_last( group8<T1,T2,T3,T4,T5,T6,T7,T8> const& x) -{ - return group1<T8> (x.a8_); -} - - - -template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9> -inline -group8<T1,T2,T3,T4,T5,T6,T7,T8> -group_head( group9<T1,T2,T3,T4,T5,T6,T7,T8,T9> const& x) -{ - return group8<T1,T2,T3,T4,T5,T6,T7,T8> (x.a1_,x.a2_,x.a3_,x.a4_,x.a5_,x.a6_,x.a7_,x.a8_); -} - -template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9> -inline -group1<T9> -group_last( group9<T1,T2,T3,T4,T5,T6,T7,T8,T9> const& x) -{ - return group1<T9> (x.a9_); -} - - - -template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9,class T10> -inline -group9<T1,T2,T3,T4,T5,T6,T7,T8,T9> -group_head( group10<T1,T2,T3,T4,T5,T6,T7,T8,T9,T10> const& x) -{ - return group9<T1,T2,T3,T4,T5,T6,T7,T8,T9> (x.a1_,x.a2_,x.a3_,x.a4_,x.a5_,x.a6_,x.a7_,x.a8_,x.a9_); -} - -template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9,class T10> -inline -group1<T10> -group_last( group10<T1,T2,T3,T4,T5,T6,T7,T8,T9,T10> const& x) -{ - return group1<T10> (x.a10_); -} - - - - - -} // namespace detail - - - -// helper functions - - -inline detail::group1< detail::group0 > -group() { return detail::group1< detail::group0 > ( detail::group0() ); } - -template <class T1, class Var> -inline -detail::group1< detail::group2<T1, Var const&> > - group(T1 a1, Var const& var) -{ - return detail::group1< detail::group2<T1, Var const&> > - ( detail::group2<T1, Var const&> - (a1, var) - ); -} - -template <class T1,class T2, class Var> -inline -detail::group1< detail::group3<T1,T2, Var const&> > - group(T1 a1,T2 a2, Var const& var) -{ - return detail::group1< detail::group3<T1,T2, Var const&> > - ( detail::group3<T1,T2, Var const&> - (a1,a2, var) - ); -} - -template <class T1,class T2,class T3, class Var> -inline -detail::group1< detail::group4<T1,T2,T3, Var const&> > - group(T1 a1,T2 a2,T3 a3, Var const& var) -{ - return detail::group1< detail::group4<T1,T2,T3, Var const&> > - ( detail::group4<T1,T2,T3, Var const&> - (a1,a2,a3, var) - ); -} - -template <class T1,class T2,class T3,class T4, class Var> -inline -detail::group1< detail::group5<T1,T2,T3,T4, Var const&> > - group(T1 a1,T2 a2,T3 a3,T4 a4, Var const& var) -{ - return detail::group1< detail::group5<T1,T2,T3,T4, Var const&> > - ( detail::group5<T1,T2,T3,T4, Var const&> - (a1,a2,a3,a4, var) - ); -} - -template <class T1,class T2,class T3,class T4,class T5, class Var> -inline -detail::group1< detail::group6<T1,T2,T3,T4,T5, Var const&> > - group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5, Var const& var) -{ - return detail::group1< detail::group6<T1,T2,T3,T4,T5, Var const&> > - ( detail::group6<T1,T2,T3,T4,T5, Var const&> - (a1,a2,a3,a4,a5, var) - ); -} - -template <class T1,class T2,class T3,class T4,class T5,class T6, class Var> -inline -detail::group1< detail::group7<T1,T2,T3,T4,T5,T6, Var const&> > - group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6, Var const& var) -{ - return detail::group1< detail::group7<T1,T2,T3,T4,T5,T6, Var const&> > - ( detail::group7<T1,T2,T3,T4,T5,T6, Var const&> - (a1,a2,a3,a4,a5,a6, var) - ); -} - -template <class T1,class T2,class T3,class T4,class T5,class T6,class T7, class Var> -inline -detail::group1< detail::group8<T1,T2,T3,T4,T5,T6,T7, Var const&> > - group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7, Var const& var) -{ - return detail::group1< detail::group8<T1,T2,T3,T4,T5,T6,T7, Var const&> > - ( detail::group8<T1,T2,T3,T4,T5,T6,T7, Var const&> - (a1,a2,a3,a4,a5,a6,a7, var) - ); -} - -template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8, class Var> -inline -detail::group1< detail::group9<T1,T2,T3,T4,T5,T6,T7,T8, Var const&> > - group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7,T8 a8, Var const& var) -{ - return detail::group1< detail::group9<T1,T2,T3,T4,T5,T6,T7,T8, Var const&> > - ( detail::group9<T1,T2,T3,T4,T5,T6,T7,T8, Var const&> - (a1,a2,a3,a4,a5,a6,a7,a8, var) - ); -} - -template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9, class Var> -inline -detail::group1< detail::group10<T1,T2,T3,T4,T5,T6,T7,T8,T9, Var const&> > - group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7,T8 a8,T9 a9, Var const& var) -{ - return detail::group1< detail::group10<T1,T2,T3,T4,T5,T6,T7,T8,T9, Var const&> > - ( detail::group10<T1,T2,T3,T4,T5,T6,T7,T8,T9, Var const&> - (a1,a2,a3,a4,a5,a6,a7,a8,a9, var) - ); -} - - -#ifndef BOOST_NO_OVERLOAD_FOR_NON_CONST - -template <class T1, class Var> -inline -detail::group1< detail::group2<T1, Var&> > - group(T1 a1, Var& var) -{ - return detail::group1< detail::group2<T1, Var&> > - ( detail::group2<T1, Var&> - (a1, var) - ); -} - -template <class T1,class T2, class Var> -inline -detail::group1< detail::group3<T1,T2, Var&> > - group(T1 a1,T2 a2, Var& var) -{ - return detail::group1< detail::group3<T1,T2, Var&> > - ( detail::group3<T1,T2, Var&> - (a1,a2, var) - ); -} - -template <class T1,class T2,class T3, class Var> -inline -detail::group1< detail::group4<T1,T2,T3, Var&> > - group(T1 a1,T2 a2,T3 a3, Var& var) -{ - return detail::group1< detail::group4<T1,T2,T3, Var&> > - ( detail::group4<T1,T2,T3, Var&> - (a1,a2,a3, var) - ); -} - -template <class T1,class T2,class T3,class T4, class Var> -inline -detail::group1< detail::group5<T1,T2,T3,T4, Var&> > - group(T1 a1,T2 a2,T3 a3,T4 a4, Var& var) -{ - return detail::group1< detail::group5<T1,T2,T3,T4, Var&> > - ( detail::group5<T1,T2,T3,T4, Var&> - (a1,a2,a3,a4, var) - ); -} - -template <class T1,class T2,class T3,class T4,class T5, class Var> -inline -detail::group1< detail::group6<T1,T2,T3,T4,T5, Var&> > - group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5, Var& var) -{ - return detail::group1< detail::group6<T1,T2,T3,T4,T5, Var&> > - ( detail::group6<T1,T2,T3,T4,T5, Var&> - (a1,a2,a3,a4,a5, var) - ); -} - -template <class T1,class T2,class T3,class T4,class T5,class T6, class Var> -inline -detail::group1< detail::group7<T1,T2,T3,T4,T5,T6, Var&> > - group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6, Var& var) -{ - return detail::group1< detail::group7<T1,T2,T3,T4,T5,T6, Var&> > - ( detail::group7<T1,T2,T3,T4,T5,T6, Var&> - (a1,a2,a3,a4,a5,a6, var) - ); -} - -template <class T1,class T2,class T3,class T4,class T5,class T6,class T7, class Var> -inline -detail::group1< detail::group8<T1,T2,T3,T4,T5,T6,T7, Var&> > - group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7, Var& var) -{ - return detail::group1< detail::group8<T1,T2,T3,T4,T5,T6,T7, Var&> > - ( detail::group8<T1,T2,T3,T4,T5,T6,T7, Var&> - (a1,a2,a3,a4,a5,a6,a7, var) - ); -} - -template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8, class Var> -inline -detail::group1< detail::group9<T1,T2,T3,T4,T5,T6,T7,T8, Var&> > - group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7,T8 a8, Var& var) -{ - return detail::group1< detail::group9<T1,T2,T3,T4,T5,T6,T7,T8, Var&> > - ( detail::group9<T1,T2,T3,T4,T5,T6,T7,T8, Var&> - (a1,a2,a3,a4,a5,a6,a7,a8, var) - ); -} - -template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9, class Var> -inline -detail::group1< detail::group10<T1,T2,T3,T4,T5,T6,T7,T8,T9, Var&> > - group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7,T8 a8,T9 a9, Var& var) -{ - return detail::group1< detail::group10<T1,T2,T3,T4,T5,T6,T7,T8,T9, Var&> > - ( detail::group10<T1,T2,T3,T4,T5,T6,T7,T8,T9, Var&> - (a1,a2,a3,a4,a5,a6,a7,a8,a9, var) - ); -} - - -#endif //end- #ifndef BOOST_NO_OVERLOAD_FOR_NON_CONST - - -} // namespace io - -} // namespace boost - - -#endif // BOOST_FORMAT_GROUP_HPP diff --git a/src/boost/format/internals.hpp b/src/boost/format/internals.hpp deleted file mode 100644 index d25eb4c864..0000000000 --- a/src/boost/format/internals.hpp +++ /dev/null @@ -1,167 +0,0 @@ -// -*- C++ -*- -// Boost general library 'format' --------------------------- -// See http://www.boost.org for updates, documentation, and revision history. - -// (C) Samuel Krempp 2001 -// krempp@crans.ens-cachan.fr -// Permission to copy, use, modify, sell and -// distribute this software is granted provided this copyright notice appears -// in all copies. This software is provided "as is" without express or implied -// warranty, and with no claim as to its suitability for any purpose. - -// ideas taken from Rüdiger Loos's format class -// and Karl Nelson's ofstream - -// ---------------------------------------------------------------------------- -// internals.hpp : internal structs. included by format.hpp -// stream_format_state, and format_item -// ---------------------------------------------------------------------------- - - -#ifndef BOOST_FORMAT_INTERNALS_HPP -#define BOOST_FORMAT_INTERNALS_HPP - - -#include <string> -#include <sstream> - -namespace boost { -namespace io { -namespace detail { - - -// -------------- -// set of params that define the format state of a stream - -struct stream_format_state -{ - typedef std::ios basic_ios; - - std::streamsize width_; - std::streamsize precision_; - char fill_; - std::ios::fmtflags flags_; - - stream_format_state() : width_(-1), precision_(-1), fill_(0), flags_(std::ios::dec) {} - stream_format_state(basic_ios& os) {set_by_stream(os); } - - void apply_on(basic_ios & os) const; //- applies format_state to the stream - template<class T> void apply_manip(T manipulator) //- modifies state by applying manipulator. - { apply_manip_body<T>( *this, manipulator) ; } - void reset(); //- sets to default state. - void set_by_stream(const basic_ios& os); //- sets to os's state. -}; - - - -// -------------- -// format_item : stores all parameters that can be defined by directives in the format-string - -struct format_item -{ - enum pad_values { zeropad = 1, spacepad =2, centered=4, tabulation = 8 }; - - enum arg_values { argN_no_posit = -1, // non-positional directive. argN will be set later. - argN_tabulation = -2, // tabulation directive. (no argument read) - argN_ignored = -3 // ignored directive. (no argument read) - }; - typedef BOOST_IO_STD ios basic_ios; - typedef detail::stream_format_state stream_format_state; - typedef std::string string_t; - typedef BOOST_IO_STD ostringstream internal_stream_t; - - - int argN_; //- argument number (starts at 0, eg : %1 => argN=0) - // negative values are used for items that don't process - // an argument - string_t res_; //- result of the formatting of this item - string_t appendix_; //- piece of string between this item and the next - - stream_format_state ref_state_;// set by parsing the format_string, is only affected by modify_item - stream_format_state state_; // always same as ref_state, _unless_ modified by manipulators 'group(..)' - - // non-stream format-state parameters - signed int truncate_; //- is >=0 for directives like %.5s (take 5 chars from the string) - unsigned int pad_scheme_; //- several possible padding schemes can mix. see pad_values - - format_item() : argN_(argN_no_posit), truncate_(-1), pad_scheme_(0) {} - - void compute_states(); // sets states according to truncate and pad_scheme. -}; - - - -// ----------------------------------------------------------- -// Definitions -// ----------------------------------------------------------- - -// --- stream_format_state:: ------------------------------------------- -inline -void stream_format_state::apply_on(basic_ios & os) const - // set the state of this stream according to our params -{ - if(width_ != -1) - os.width(width_); - if(precision_ != -1) - os.precision(precision_); - if(fill_ != 0) - os.fill(fill_); - os.flags(flags_); -} - -inline -void stream_format_state::set_by_stream(const basic_ios& os) - // set our params according to the state of this stream -{ - flags_ = os.flags(); - width_ = os.width(); - precision_ = os.precision(); - fill_ = os.fill(); -} - -template<class T> inline -void apply_manip_body( stream_format_state& self, - T manipulator) - // modify our params according to the manipulator -{ - BOOST_IO_STD stringstream ss; - self.apply_on( ss ); - ss << manipulator; - self.set_by_stream( ss ); -} - -inline -void stream_format_state::reset() - // set our params to standard's default state -{ - width_=-1; precision_=-1; fill_=0; - flags_ = std::ios::dec; -} - - -// --- format_items:: ------------------------------------------- -inline -void format_item::compute_states() - // reflect pad_scheme_ on state_ and ref_state_ - // because some pad_schemes has complex consequences on several state params. -{ - if(pad_scheme_ & zeropad) - { - if(ref_state_.flags_ & std::ios::left) - { - pad_scheme_ = pad_scheme_ & (~zeropad); // ignore zeropad in left alignment - } - else - { - ref_state_.fill_='0'; - ref_state_.flags_ |= std::ios::internal; - } - } - state_ = ref_state_; -} - - -} } } // namespaces boost :: io :: detail - - -#endif // BOOST_FORMAT_INTERNALS_HPP diff --git a/src/boost/format/internals_fwd.hpp b/src/boost/format/internals_fwd.hpp deleted file mode 100644 index a8ebf7c3ab..0000000000 --- a/src/boost/format/internals_fwd.hpp +++ /dev/null @@ -1,65 +0,0 @@ -// -*- C++ -*- -// Boost general library 'format' --------------------------- -// See http://www.boost.org for updates, documentation, and revision history. - -// (C) Samuel Krempp 2001 -// krempp@crans.ens-cachan.fr -// Permission to copy, use, modify, sell and -// distribute this software is granted provided this copyright notice appears -// in all copies. This software is provided "as is" without express or implied -// warranty, and with no claim as to its suitability for any purpose. - -// ideas taken from Rüdiger Loos's format class -// and Karl Nelson's ofstream (also took its parsing code as basis for printf parsing) - -// ------------------------------------------------------------------------------ -// internals_fwd.hpp : forward declarations, for internal headers -// ------------------------------------------------------------------------------ - -#ifndef BOOST_FORMAT_INTERNAL_FWD_HPP -#define BOOST_FORMAT_INTERNAL_FWD_HPP - -#include "boost/format/format_fwd.hpp" - - -namespace boost { -namespace io { - -namespace detail { - struct stream_format_state; - struct format_item; -} - - -namespace detail { - - // these functions were intended as methods, - // but MSVC have problems with template member functions : - - // defined in format_implementation.hpp : - template<class T> - basic_format& modify_item_body( basic_format& self, - int itemN, const T& manipulator); - - template<class T> - basic_format& bind_arg_body( basic_format& self, - int argN, const T& val); - - template<class T> - void apply_manip_body( stream_format_state& self, - T manipulator); - - // argument feeding (defined in feed_args.hpp ) : - template<class T> - void distribute(basic_format& self, T x); - - template<class T> - basic_format& feed(basic_format& self, T x); - -} // namespace detail - -} // namespace io -} // namespace boost - - -#endif // BOOST_FORMAT_INTERNAL_FWD_HPP diff --git a/src/boost/format/local.mk b/src/boost/format/local.mk deleted file mode 100644 index 3776eff382..0000000000 --- a/src/boost/format/local.mk +++ /dev/null @@ -1,7 +0,0 @@ -libraries += libformat - -libformat_NAME = libnixformat - -libformat_DIR := $(d) - -libformat_SOURCES := $(wildcard $(d)/*.cc) diff --git a/src/boost/format/macros_default.hpp b/src/boost/format/macros_default.hpp deleted file mode 100644 index 4fd84a163f..0000000000 --- a/src/boost/format/macros_default.hpp +++ /dev/null @@ -1,48 +0,0 @@ -// -*- C++ -*- -// Boost general library 'format' --------------------------- -// See http://www.boost.org for updates, documentation, and revision history. - -// (C) Samuel Krempp 2001 -// krempp@crans.ens-cachan.fr -// Permission to copy, use, modify, sell and -// distribute this software is granted provided this copyright notice appears -// in all copies. This software is provided "as is" without express or implied -// warranty, and with no claim as to its suitability for any purpose. - -// ideas taken from Rüdiger Loos's format class -// and Karl Nelson's ofstream (also took its parsing code as basis for printf parsing) - -// ------------------------------------------------------------------------------ -// macros_default.hpp : configuration for the format library -// provides default values for the stl workaround macros -// ------------------------------------------------------------------------------ - -#ifndef BOOST_FORMAT_MACROS_DEFAULT_HPP -#define BOOST_FORMAT_MACROS_DEFAULT_HPP - -// *** This should go to "boost/config/suffix.hpp". - -#ifndef BOOST_IO_STD -# define BOOST_IO_STD std:: -#endif - -// **** Workaround for io streams, stlport and msvc. -#ifdef BOOST_IO_NEEDS_USING_DECLARATION -namespace boost { - using std::char_traits; - using std::basic_ostream; - using std::basic_ostringstream; - namespace io { - using std::basic_ostream; - namespace detail { - using std::basic_ios; - using std::basic_ostream; - using std::basic_ostringstream; - } - } -} -#endif - -// ------------------------------------------------------------------------------ - -#endif // BOOST_FORMAT_MACROS_DEFAULT_HPP diff --git a/src/boost/format/parsing.cc b/src/boost/format/parsing.cc deleted file mode 100644 index 34c36adeb7..0000000000 --- a/src/boost/format/parsing.cc +++ /dev/null @@ -1,454 +0,0 @@ -// -*- C++ -*- -// Boost general library 'format' --------------------------- -// See http://www.boost.org for updates, documentation, and revision history. - -// (C) Samuel Krempp 2001 -// krempp@crans.ens-cachan.fr -// Permission to copy, use, modify, sell and -// distribute this software is granted provided this copyright notice appears -// in all copies. This software is provided "as is" without express or implied -// warranty, and with no claim as to its suitability for any purpose. - -// ideas taken from Rudiger Loos's format class -// and Karl Nelson's ofstream (also took its parsing code as basis for printf parsing) - -// ------------------------------------------------------------------------------ -// parsing.hpp : implementation of the parsing member functions -// ( parse, parse_printf_directive) -// ------------------------------------------------------------------------------ - - -#ifndef BOOST_FORMAT_PARSING_HPP -#define BOOST_FORMAT_PARSING_HPP - - -#include <boost/format.hpp> -#include <boost/throw_exception.hpp> -#include <boost/assert.hpp> - - -namespace boost { -namespace io { -namespace detail { - - template<class Stream> inline - bool wrap_isdigit(char c, Stream &os) - { -#ifndef BOOST_NO_LOCALE_ISIDIGIT - return std::isdigit(c, os.rdbuf()->getloc() ); -# else - using namespace std; - return isdigit(c); -#endif - } //end- wrap_isdigit(..) - - template<class Res> inline - Res str2int(const std::string& s, - std::string::size_type start, - BOOST_IO_STD ios &os, - const Res = Res(0) ) - // Input : char string, with starting index - // a basic_ios& merely to call its widen/narrow member function in the desired locale. - // Effects : reads s[start:] and converts digits into an integral n, of type Res - // Returns : n - { - Res n = 0; - while(start<s.size() && wrap_isdigit(s[start], os) ) { - char cur_ch = s[start]; - BOOST_ASSERT(cur_ch != 0 ); // since we called isdigit, this should not happen. - n *= 10; - n += cur_ch - '0'; // 22.2.1.1.2 of the C++ standard - ++start; - } - return n; - } - - void skip_asterisk(const std::string & buf, - std::string::size_type * pos_p, - BOOST_IO_STD ios &os) - // skip printf's "asterisk-fields" directives in the format-string buf - // Input : char string, with starting index *pos_p - // a basic_ios& merely to call its widen/narrow member function in the desired locale. - // Effects : advance *pos_p by skipping printf's asterisk fields. - // Returns : nothing - { - using namespace std; - BOOST_ASSERT( pos_p != 0); - if(*pos_p >= buf.size() ) return; - if(buf[ *pos_p]=='*') { - ++ (*pos_p); - while (*pos_p < buf.size() && wrap_isdigit(buf[*pos_p],os)) ++(*pos_p); - if(buf[*pos_p]=='$') ++(*pos_p); - } - } - - - inline void maybe_throw_exception( unsigned char exceptions) - // auxiliary func called by parse_printf_directive - // for centralising error handling - // it either throws if user sets the corresponding flag, or does nothing. - { - if(exceptions & io::bad_format_string_bit) - boost::throw_exception(io::bad_format_string()); - } - - - - bool parse_printf_directive(const std::string & buf, - std::string::size_type * pos_p, - detail::format_item * fpar, - BOOST_IO_STD ios &os, - unsigned char exceptions) - // Input : a 'printf-directive' in the format-string, starting at buf[ *pos_p ] - // a basic_ios& merely to call its widen/narrow member function in the desired locale. - // a bitset'excpetions' telling whether to throw exceptions on errors. - // Returns : true if parse somehow succeeded (possibly ignoring errors if exceptions disabled) - // false if it failed so bad that the directive should be printed verbatim - // Effects : - *pos_p is incremented so that buf[*pos_p] is the first char after the directive - // - *fpar is set with the parameters read in the directive - { - typedef format_item format_item_t; - BOOST_ASSERT( pos_p != 0); - std::string::size_type &i1 = *pos_p, - i0; - fpar->argN_ = format_item_t::argN_no_posit; // if no positional-directive - - bool in_brackets=false; - if(buf[i1]=='|') - { - in_brackets=true; - if( ++i1 >= buf.size() ) { - maybe_throw_exception(exceptions); - return false; - } - } - - // the flag '0' would be picked as a digit for argument order, but here it's a flag : - if(buf[i1]=='0') - goto parse_flags; - - // handle argument order (%2$d) or possibly width specification: %2d - i0 = i1; // save position before digits - while (i1 < buf.size() && wrap_isdigit(buf[i1], os)) - ++i1; - if (i1!=i0) - { - if( i1 >= buf.size() ) { - maybe_throw_exception(exceptions); - return false; - } - int n=str2int(buf,i0, os, int(0) ); - - // %N% case : this is already the end of the directive - if( buf[i1] == '%' ) - { - fpar->argN_ = n-1; - ++i1; - if( in_brackets) - maybe_throw_exception(exceptions); - // but don't return. maybe "%" was used in lieu of '$', so we go on. - else return true; - } - - if ( buf[i1]=='$' ) - { - fpar->argN_ = n-1; - ++i1; - } - else - { - // non-positionnal directive - fpar->ref_state_.width_ = n; - fpar->argN_ = format_item_t::argN_no_posit; - goto parse_precision; - } - } - - parse_flags: - // handle flags - while ( i1 <buf.size()) // as long as char is one of + - = # 0 l h or ' ' - { - // misc switches - switch (buf[i1]) - { - case '\'' : break; // no effect yet. (painful to implement) - case 'l': - case 'h': // short/long modifier : for printf-comaptibility (no action needed) - break; - case '-': - fpar->ref_state_.flags_ |= std::ios::left; - break; - case '=': - fpar->pad_scheme_ |= format_item_t::centered; - break; - case ' ': - fpar->pad_scheme_ |= format_item_t::spacepad; - break; - case '+': - fpar->ref_state_.flags_ |= std::ios::showpos; - break; - case '0': - fpar->pad_scheme_ |= format_item_t::zeropad; - // need to know alignment before really setting flags, - // so just add 'zeropad' flag for now, it will be processed later. - break; - case '#': - fpar->ref_state_.flags_ |= std::ios::showpoint | std::ios::showbase; - break; - default: - goto parse_width; - } - ++i1; - } // loop on flag. - if( i1>=buf.size()) { - maybe_throw_exception(exceptions); - return true; - } - - parse_width: - // handle width spec - skip_asterisk(buf, &i1, os); // skips 'asterisk fields' : *, or *N$ - i0 = i1; // save position before digits - while (i1<buf.size() && wrap_isdigit(buf[i1], os)) - i1++; - - if (i1!=i0) - { fpar->ref_state_.width_ = str2int( buf,i0, os, std::streamsize(0) ); } - - parse_precision: - if( i1>=buf.size()) { - maybe_throw_exception(exceptions); - return true; - } - // handle precision spec - if (buf[i1]=='.') - { - ++i1; - skip_asterisk(buf, &i1, os); - i0 = i1; // save position before digits - while (i1<buf.size() && wrap_isdigit(buf[i1], os)) - ++i1; - - if(i1==i0) - fpar->ref_state_.precision_ = 0; - else - fpar->ref_state_.precision_ = str2int(buf,i0, os, std::streamsize(0) ); - } - - // handle formatting-type flags : - while( i1<buf.size() && - ( buf[i1]=='l' || buf[i1]=='L' || buf[i1]=='h') ) - ++i1; - if( i1>=buf.size()) { - maybe_throw_exception(exceptions); - return true; - } - - if( in_brackets && buf[i1]=='|' ) - { - ++i1; - return true; - } - switch (buf[i1]) - { - case 'X': - fpar->ref_state_.flags_ |= std::ios::uppercase; - case 'p': // pointer => set hex. - case 'x': - fpar->ref_state_.flags_ &= ~std::ios::basefield; - fpar->ref_state_.flags_ |= std::ios::hex; - break; - - case 'o': - fpar->ref_state_.flags_ &= ~std::ios::basefield; - fpar->ref_state_.flags_ |= std::ios::oct; - break; - - case 'E': - fpar->ref_state_.flags_ |= std::ios::uppercase; - case 'e': - fpar->ref_state_.flags_ &= ~std::ios::floatfield; - fpar->ref_state_.flags_ |= std::ios::scientific; - - fpar->ref_state_.flags_ &= ~std::ios::basefield; - fpar->ref_state_.flags_ |= std::ios::dec; - break; - - case 'f': - fpar->ref_state_.flags_ &= ~std::ios::floatfield; - fpar->ref_state_.flags_ |= std::ios::fixed; - case 'u': - case 'd': - case 'i': - fpar->ref_state_.flags_ &= ~std::ios::basefield; - fpar->ref_state_.flags_ |= std::ios::dec; - break; - - case 'T': - ++i1; - if( i1 >= buf.size()) - maybe_throw_exception(exceptions); - else - fpar->ref_state_.fill_ = buf[i1]; - fpar->pad_scheme_ |= format_item_t::tabulation; - fpar->argN_ = format_item_t::argN_tabulation; - break; - case 't': - fpar->ref_state_.fill_ = ' '; - fpar->pad_scheme_ |= format_item_t::tabulation; - fpar->argN_ = format_item_t::argN_tabulation; - break; - - case 'G': - fpar->ref_state_.flags_ |= std::ios::uppercase; - break; - case 'g': // 'g' conversion is default for floats. - fpar->ref_state_.flags_ &= ~std::ios::basefield; - fpar->ref_state_.flags_ |= std::ios::dec; - - // CLEAR all floatield flags, so stream will CHOOSE - fpar->ref_state_.flags_ &= ~std::ios::floatfield; - break; - - case 'C': - case 'c': - fpar->truncate_ = 1; - break; - case 'S': - case 's': - fpar->truncate_ = fpar->ref_state_.precision_; - fpar->ref_state_.precision_ = -1; - break; - case 'n' : - fpar->argN_ = format_item_t::argN_ignored; - break; - default: - maybe_throw_exception(exceptions); - } - ++i1; - - if( in_brackets ) - { - if( i1<buf.size() && buf[i1]=='|' ) - { - ++i1; - return true; - } - else maybe_throw_exception(exceptions); - } - return true; - } - -} // detail namespace -} // io namespace - - -// ----------------------------------------------- -// format :: parse(..) - -void basic_format::parse(const string_t & buf) - // parse the format-string -{ - using namespace std; - const char arg_mark = '%'; - bool ordered_args=true; - int max_argN=-1; - string_t::size_type i1=0; - int num_items=0; - - // A: find upper_bound on num_items and allocates arrays - i1=0; - while( (i1=buf.find(arg_mark,i1)) != string::npos ) - { - if( i1+1 >= buf.size() ) { - if(exceptions() & io::bad_format_string_bit) - boost::throw_exception(io::bad_format_string()); // must not end in "bla bla %" - else break; // stop there, ignore last '%' - } - if(buf[i1+1] == buf[i1] ) { i1+=2; continue; } // escaped "%%" / "##" - ++i1; - - // in case of %N% directives, dont count it double (wastes allocations..) : - while(i1 < buf.size() && io::detail::wrap_isdigit(buf[i1],oss_)) ++i1; - if( i1 < buf.size() && buf[i1] == arg_mark ) ++ i1; - - ++num_items; - } - items_.assign( num_items, format_item_t() ); - - // B: Now the real parsing of the format string : - num_items=0; - i1 = 0; - string_t::size_type i0 = i1; - bool special_things=false; - int cur_it=0; - while( (i1=buf.find(arg_mark,i1)) != string::npos ) - { - string_t & piece = (cur_it==0) ? prefix_ : items_[cur_it-1].appendix_; - - if( buf[i1+1] == buf[i1] ) // escaped mark, '%%' - { - piece += buf.substr(i0, i1-i0) + buf[i1]; - i1+=2; i0=i1; - continue; - } - BOOST_ASSERT( static_cast<unsigned int>(cur_it) < items_.size() || cur_it==0); - - if(i1!=i0) piece += buf.substr(i0, i1-i0); - ++i1; - - bool parse_ok; - parse_ok = io::detail::parse_printf_directive(buf, &i1, &items_[cur_it], oss_, exceptions()); - if( ! parse_ok ) continue; // the directive will be printed verbatim - - i0=i1; - items_[cur_it].compute_states(); // process complex options, like zeropad, into stream params. - - int argN=items_[cur_it].argN_; - if(argN == format_item_t::argN_ignored) - continue; - if(argN ==format_item_t::argN_no_posit) - ordered_args=false; - else if(argN == format_item_t::argN_tabulation) special_things=true; - else if(argN > max_argN) max_argN = argN; - ++num_items; - ++cur_it; - } // loop on %'s - BOOST_ASSERT(cur_it == num_items); - - // store the final piece of string - string_t & piece = (cur_it==0) ? prefix_ : items_[cur_it-1].appendix_; - piece += buf.substr(i0); - - if( !ordered_args) - { - if(max_argN >= 0 ) // dont mix positional with non-positionnal directives - { - if(exceptions() & io::bad_format_string_bit) - boost::throw_exception(io::bad_format_string()); - // else do nothing. => positionnal arguments are processed as non-positionnal - } - // set things like it would have been with positional directives : - int non_ordered_items = 0; - for(int i=0; i< num_items; ++i) - if(items_[i].argN_ == format_item_t::argN_no_posit) - { - items_[i].argN_ = non_ordered_items; - ++non_ordered_items; - } - max_argN = non_ordered_items-1; - } - - // C: set some member data : - items_.resize(num_items); - - if(special_things) style_ |= special_needs; - num_args_ = max_argN + 1; - if(ordered_args) style_ |= ordered; - else style_ &= ~ordered; -} - -} // namespace boost - - -#endif // BOOST_FORMAT_PARSING_HPP diff --git a/src/boost/throw_exception.hpp b/src/boost/throw_exception.hpp deleted file mode 100644 index 07b4ae5cea..0000000000 --- a/src/boost/throw_exception.hpp +++ /dev/null @@ -1,47 +0,0 @@ -#ifndef BOOST_THROW_EXCEPTION_HPP_INCLUDED -#define BOOST_THROW_EXCEPTION_HPP_INCLUDED - -// MS compatible compilers support #pragma once - -#if defined(_MSC_VER) && (_MSC_VER >= 1020) -# pragma once -#endif - -// -// boost/throw_exception.hpp -// -// Copyright (c) 2002 Peter Dimov and Multi Media Ltd. -// -// Permission to copy, use, modify, sell and distribute this software -// is granted provided this copyright notice appears in all copies. -// This software is provided "as is" without express or implied -// warranty, and with no claim as to its suitability for any purpose. -// -// http://www.boost.org/libs/utility/throw_exception.html -// - -//#include <boost/config.hpp> - -#ifdef BOOST_NO_EXCEPTIONS -# include <exception> -#endif - -namespace boost -{ - -#ifdef BOOST_NO_EXCEPTIONS - -void throw_exception(std::exception const & e); // user defined - -#else - -template<class E> void throw_exception(E const & e) -{ - throw e; -} - -#endif - -} // namespace boost - -#endif // #ifndef BOOST_THROW_EXCEPTION_HPP_INCLUDED diff --git a/src/bsdiff-4.3/bsdiff.1 b/src/bsdiff-4.3/bsdiff.1 deleted file mode 100644 index ead6c4deb5..0000000000 --- a/src/bsdiff-4.3/bsdiff.1 +++ /dev/null @@ -1,63 +0,0 @@ -.\"- -.\" Copyright 2003-2005 Colin Percival -.\" All rights reserved -.\" -.\" Redistribution and use in source and binary forms, with or without -.\" modification, are permitted providing that the following conditions -.\" are met: -.\" 1. Redistributions of source code must retain the above copyright -.\" notice, this list of conditions and the following disclaimer. -.\" 2. Redistributions in binary form must reproduce the above copyright -.\" notice, this list of conditions and the following disclaimer in the -.\" documentation and/or other materials provided with the distribution. -.\" -.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR -.\" IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -.\" WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY -.\" DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -.\" STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING -.\" IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -.\" POSSIBILITY OF SUCH DAMAGE. -.\" -.\" $FreeBSD: src/usr.bin/bsdiff/bsdiff/bsdiff.1,v 1.1 2005/08/06 01:59:05 cperciva Exp $ -.\" -.Dd May 18, 2003 -.Dt BSDIFF 1 -.Os FreeBSD -.Sh NAME -.Nm bsdiff -.Nd generate a patch between two binary files -.Sh SYNOPSIS -.Nm -.Ao Ar oldfile Ac Ao Ar newfile Ac Ao Ar patchfile Ac -.Sh DESCRIPTION -.Nm -compares -.Ao Ar oldfile Ac -to -.Ao Ar newfile Ac -and writes to -.Ao Ar patchfile Ac -a binary patch suitable for use by bspatch(1). -When -.Ao Ar oldfile Ac -and -.Ao Ar newfile Ac -are two versions of an executable program, the -patches produced are on average a factor of five smaller -than those produced by any other binary patch tool known -to the author. -.Pp -.Nm -uses memory equal to 17 times the size of -.Ao Ar oldfile Ac , -and requires -an absolute minimum working set size of 8 times the size of oldfile. -.Sh SEE ALSO -.Xr bspatch 1 -.Sh AUTHORS -.An Colin Percival Aq cperciva@freebsd.org diff --git a/src/bsdiff-4.3/bsdiff.c b/src/bsdiff-4.3/bsdiff.c deleted file mode 100644 index 374ed038fa..0000000000 --- a/src/bsdiff-4.3/bsdiff.c +++ /dev/null @@ -1,405 +0,0 @@ -/*- - * Copyright 2003-2005 Colin Percival - * All rights reserved - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted providing that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING - * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#if 0 -__FBSDID("$FreeBSD: src/usr.bin/bsdiff/bsdiff/bsdiff.c,v 1.1 2005/08/06 01:59:05 cperciva Exp $"); -#endif - -#include <sys/types.h> - -#include <bzlib.h> -#include <err.h> -#include <fcntl.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <unistd.h> - -#define MIN(x,y) (((x)<(y)) ? (x) : (y)) - -static void split(off_t *I,off_t *V,off_t start,off_t len,off_t h) -{ - off_t i,j,k,x,tmp,jj,kk; - - if(len<16) { - for(k=start;k<start+len;k+=j) { - j=1;x=V[I[k]+h]; - for(i=1;k+i<start+len;i++) { - if(V[I[k+i]+h]<x) { - x=V[I[k+i]+h]; - j=0; - }; - if(V[I[k+i]+h]==x) { - tmp=I[k+j];I[k+j]=I[k+i];I[k+i]=tmp; - j++; - }; - }; - for(i=0;i<j;i++) V[I[k+i]]=k+j-1; - if(j==1) I[k]=-1; - }; - return; - }; - - x=V[I[start+len/2]+h]; - jj=0;kk=0; - for(i=start;i<start+len;i++) { - if(V[I[i]+h]<x) jj++; - if(V[I[i]+h]==x) kk++; - }; - jj+=start;kk+=jj; - - i=start;j=0;k=0; - while(i<jj) { - if(V[I[i]+h]<x) { - i++; - } else if(V[I[i]+h]==x) { - tmp=I[i];I[i]=I[jj+j];I[jj+j]=tmp; - j++; - } else { - tmp=I[i];I[i]=I[kk+k];I[kk+k]=tmp; - k++; - }; - }; - - while(jj+j<kk) { - if(V[I[jj+j]+h]==x) { - j++; - } else { - tmp=I[jj+j];I[jj+j]=I[kk+k];I[kk+k]=tmp; - k++; - }; - }; - - if(jj>start) split(I,V,start,jj-start,h); - - for(i=0;i<kk-jj;i++) V[I[jj+i]]=kk-1; - if(jj==kk-1) I[jj]=-1; - - if(start+len>kk) split(I,V,kk,start+len-kk,h); -} - -static void qsufsort(off_t *I,off_t *V,u_char *old,off_t oldsize) -{ - off_t buckets[256]; - off_t i,h,len; - - for(i=0;i<256;i++) buckets[i]=0; - for(i=0;i<oldsize;i++) buckets[old[i]]++; - for(i=1;i<256;i++) buckets[i]+=buckets[i-1]; - for(i=255;i>0;i--) buckets[i]=buckets[i-1]; - buckets[0]=0; - - for(i=0;i<oldsize;i++) I[++buckets[old[i]]]=i; - I[0]=oldsize; - for(i=0;i<oldsize;i++) V[i]=buckets[old[i]]; - V[oldsize]=0; - for(i=1;i<256;i++) if(buckets[i]==buckets[i-1]+1) I[buckets[i]]=-1; - I[0]=-1; - - for(h=1;I[0]!=-(oldsize+1);h+=h) { - len=0; - for(i=0;i<oldsize+1;) { - if(I[i]<0) { - len-=I[i]; - i-=I[i]; - } else { - if(len) I[i-len]=-len; - len=V[I[i]]+1-i; - split(I,V,i,len,h); - i+=len; - len=0; - }; - }; - if(len) I[i-len]=-len; - }; - - for(i=0;i<oldsize+1;i++) I[V[i]]=i; -} - -static off_t matchlen(u_char *old,off_t oldsize,u_char *new,off_t newsize) -{ - off_t i; - - for(i=0;(i<oldsize)&&(i<newsize);i++) - if(old[i]!=new[i]) break; - - return i; -} - -static off_t search(off_t *I,u_char *old,off_t oldsize, - u_char *new,off_t newsize,off_t st,off_t en,off_t *pos) -{ - off_t x,y; - - if(en-st<2) { - x=matchlen(old+I[st],oldsize-I[st],new,newsize); - y=matchlen(old+I[en],oldsize-I[en],new,newsize); - - if(x>y) { - *pos=I[st]; - return x; - } else { - *pos=I[en]; - return y; - } - }; - - x=st+(en-st)/2; - if(memcmp(old+I[x],new,MIN(oldsize-I[x],newsize))<0) { - return search(I,old,oldsize,new,newsize,x,en,pos); - } else { - return search(I,old,oldsize,new,newsize,st,x,pos); - }; -} - -static void offtout(off_t x,u_char *buf) -{ - off_t y; - - if(x<0) y=-x; else y=x; - - buf[0]=y%256;y-=buf[0]; - y=y/256;buf[1]=y%256;y-=buf[1]; - y=y/256;buf[2]=y%256;y-=buf[2]; - y=y/256;buf[3]=y%256;y-=buf[3]; - y=y/256;buf[4]=y%256;y-=buf[4]; - y=y/256;buf[5]=y%256;y-=buf[5]; - y=y/256;buf[6]=y%256;y-=buf[6]; - y=y/256;buf[7]=y%256; - - if(x<0) buf[7]|=0x80; -} - -int main(int argc,char *argv[]) -{ - int fd; - u_char *old,*new; - off_t oldsize,newsize; - off_t *I,*V; - off_t scan,pos,len; - off_t lastscan,lastpos,lastoffset; - off_t oldscore,scsc; - off_t s,Sf,lenf,Sb,lenb; - off_t overlap,Ss,lens; - off_t i; - off_t dblen,eblen; - u_char *db,*eb; - u_char buf[8]; - u_char header[32]; - FILE * pf; - BZFILE * pfbz2; - int bz2err; - - if(argc!=4) errx(1,"usage: %s oldfile newfile patchfile\n",argv[0]); - - /* Allocate oldsize+1 bytes instead of oldsize bytes to ensure - that we never try to malloc(0) and get a NULL pointer */ - if(((fd=open(argv[1],O_RDONLY,0))<0) || - ((oldsize=lseek(fd,0,SEEK_END))==-1) || - ((old=malloc(oldsize+1))==NULL) || - (lseek(fd,0,SEEK_SET)!=0) || - (read(fd,old,oldsize)!=oldsize) || - (close(fd)==-1)) err(1,"%s",argv[1]); - - if(((I=malloc((oldsize+1)*sizeof(off_t)))==NULL) || - ((V=malloc((oldsize+1)*sizeof(off_t)))==NULL)) err(1,NULL); - - qsufsort(I,V,old,oldsize); - - free(V); - - /* Allocate newsize+1 bytes instead of newsize bytes to ensure - that we never try to malloc(0) and get a NULL pointer */ - if(((fd=open(argv[2],O_RDONLY,0))<0) || - ((newsize=lseek(fd,0,SEEK_END))==-1) || - ((new=malloc(newsize+1))==NULL) || - (lseek(fd,0,SEEK_SET)!=0) || - (read(fd,new,newsize)!=newsize) || - (close(fd)==-1)) err(1,"%s",argv[2]); - - if(((db=malloc(newsize+1))==NULL) || - ((eb=malloc(newsize+1))==NULL)) err(1,NULL); - dblen=0; - eblen=0; - - /* Create the patch file */ - if ((pf = fopen(argv[3], "w")) == NULL) - err(1, "%s", argv[3]); - - /* Header is - 0 8 "BSDIFF40" - 8 8 length of bzip2ed ctrl block - 16 8 length of bzip2ed diff block - 24 8 length of new file */ - /* File is - 0 32 Header - 32 ?? Bzip2ed ctrl block - ?? ?? Bzip2ed diff block - ?? ?? Bzip2ed extra block */ - memcpy(header,"BSDIFF40",8); - offtout(0, header + 8); - offtout(0, header + 16); - offtout(newsize, header + 24); - if (fwrite(header, 32, 1, pf) != 1) - err(1, "fwrite(%s)", argv[3]); - - /* Compute the differences, writing ctrl as we go */ - if ((pfbz2 = BZ2_bzWriteOpen(&bz2err, pf, 9, 0, 0)) == NULL) - errx(1, "BZ2_bzWriteOpen, bz2err = %d", bz2err); - scan=0;len=0; - lastscan=0;lastpos=0;lastoffset=0; - while(scan<newsize) { - oldscore=0; - - for(scsc=scan+=len;scan<newsize;scan++) { - len=search(I,old,oldsize,new+scan,newsize-scan, - 0,oldsize,&pos); - if (len > 64 * 1024) break; - - for(;scsc<scan+len;scsc++) - if((scsc+lastoffset<oldsize) && - (old[scsc+lastoffset] == new[scsc])) - oldscore++; - - if(((len==oldscore) && (len!=0)) || - (len>oldscore+8)) break; - - if((scan+lastoffset<oldsize) && - (old[scan+lastoffset] == new[scan])) - oldscore--; - }; - - if((len!=oldscore) || (scan==newsize)) { - s=0;Sf=0;lenf=0; - for(i=0;(lastscan+i<scan)&&(lastpos+i<oldsize);) { - if(old[lastpos+i]==new[lastscan+i]) s++; - i++; - if(s*2-i>Sf*2-lenf) { Sf=s; lenf=i; }; - }; - - lenb=0; - if(scan<newsize) { - s=0;Sb=0; - for(i=1;(scan>=lastscan+i)&&(pos>=i);i++) { - if(old[pos-i]==new[scan-i]) s++; - if(s*2-i>Sb*2-lenb) { Sb=s; lenb=i; }; - }; - }; - - if(lastscan+lenf>scan-lenb) { - overlap=(lastscan+lenf)-(scan-lenb); - s=0;Ss=0;lens=0; - for(i=0;i<overlap;i++) { - if(new[lastscan+lenf-overlap+i]== - old[lastpos+lenf-overlap+i]) s++; - if(new[scan-lenb+i]== - old[pos-lenb+i]) s--; - if(s>Ss) { Ss=s; lens=i+1; }; - }; - - lenf+=lens-overlap; - lenb-=lens; - }; - - for(i=0;i<lenf;i++) - db[dblen+i]=new[lastscan+i]-old[lastpos+i]; - for(i=0;i<(scan-lenb)-(lastscan+lenf);i++) - eb[eblen+i]=new[lastscan+lenf+i]; - - dblen+=lenf; - eblen+=(scan-lenb)-(lastscan+lenf); - - offtout(lenf,buf); - BZ2_bzWrite(&bz2err, pfbz2, buf, 8); - if (bz2err != BZ_OK) - errx(1, "BZ2_bzWrite, bz2err = %d", bz2err); - - offtout((scan-lenb)-(lastscan+lenf),buf); - BZ2_bzWrite(&bz2err, pfbz2, buf, 8); - if (bz2err != BZ_OK) - errx(1, "BZ2_bzWrite, bz2err = %d", bz2err); - - offtout((pos-lenb)-(lastpos+lenf),buf); - BZ2_bzWrite(&bz2err, pfbz2, buf, 8); - if (bz2err != BZ_OK) - errx(1, "BZ2_bzWrite, bz2err = %d", bz2err); - - lastscan=scan-lenb; - lastpos=pos-lenb; - lastoffset=pos-scan; - }; - }; - BZ2_bzWriteClose(&bz2err, pfbz2, 0, NULL, NULL); - if (bz2err != BZ_OK) - errx(1, "BZ2_bzWriteClose, bz2err = %d", bz2err); - - /* Compute size of compressed ctrl data */ - if ((len = ftello(pf)) == -1) - err(1, "ftello"); - offtout(len-32, header + 8); - - /* Write compressed diff data */ - if ((pfbz2 = BZ2_bzWriteOpen(&bz2err, pf, 9, 0, 0)) == NULL) - errx(1, "BZ2_bzWriteOpen, bz2err = %d", bz2err); - BZ2_bzWrite(&bz2err, pfbz2, db, dblen); - if (bz2err != BZ_OK) - errx(1, "BZ2_bzWrite, bz2err = %d", bz2err); - BZ2_bzWriteClose(&bz2err, pfbz2, 0, NULL, NULL); - if (bz2err != BZ_OK) - errx(1, "BZ2_bzWriteClose, bz2err = %d", bz2err); - - /* Compute size of compressed diff data */ - if ((newsize = ftello(pf)) == -1) - err(1, "ftello"); - offtout(newsize - len, header + 16); - - /* Write compressed extra data */ - if ((pfbz2 = BZ2_bzWriteOpen(&bz2err, pf, 9, 0, 0)) == NULL) - errx(1, "BZ2_bzWriteOpen, bz2err = %d", bz2err); - BZ2_bzWrite(&bz2err, pfbz2, eb, eblen); - if (bz2err != BZ_OK) - errx(1, "BZ2_bzWrite, bz2err = %d", bz2err); - BZ2_bzWriteClose(&bz2err, pfbz2, 0, NULL, NULL); - if (bz2err != BZ_OK) - errx(1, "BZ2_bzWriteClose, bz2err = %d", bz2err); - - /* Seek to the beginning, write the header, and close the file */ - if (fseeko(pf, 0, SEEK_SET)) - err(1, "fseeko"); - if (fwrite(header, 32, 1, pf) != 1) - err(1, "fwrite(%s)", argv[3]); - if (fclose(pf)) - err(1, "fclose"); - - /* Free the memory we used */ - free(db); - free(eb); - free(I); - free(old); - free(new); - - return 0; -} diff --git a/src/bsdiff-4.3/bspatch.1 b/src/bsdiff-4.3/bspatch.1 deleted file mode 100644 index 82a2781aa7..0000000000 --- a/src/bsdiff-4.3/bspatch.1 +++ /dev/null @@ -1,59 +0,0 @@ -.\"- -.\" Copyright 2003-2005 Colin Percival -.\" All rights reserved -.\" -.\" Redistribution and use in source and binary forms, with or without -.\" modification, are permitted providing that the following conditions -.\" are met: -.\" 1. Redistributions of source code must retain the above copyright -.\" notice, this list of conditions and the following disclaimer. -.\" 2. Redistributions in binary form must reproduce the above copyright -.\" notice, this list of conditions and the following disclaimer in the -.\" documentation and/or other materials provided with the distribution. -.\" -.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR -.\" IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -.\" WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY -.\" DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -.\" STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING -.\" IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -.\" POSSIBILITY OF SUCH DAMAGE. -.\" -.\" $FreeBSD: src/usr.bin/bsdiff/bspatch/bspatch.1,v 1.1 2005/08/06 01:59:06 cperciva Exp $ -.\" -.Dd May 18, 2003 -.Dt BSPATCH 1 -.Os FreeBSD -.Sh NAME -.Nm bspatch -.Nd apply a patch built with bsdiff(1) -.Sh SYNOPSIS -.Nm -.Ao Ar oldfile Ac Ao Ar newfile Ac Ao Ar patchfile Ac -.Sh DESCRIPTION -.Nm -generates -.Ao Ar newfile Ac -from -.Ao Ar oldfile Ac -and -.Ao Ar patchfile Ac -where -.Ao Ar patchfile Ac -is a binary patch built by bsdiff(1). -.Pp -.Nm -uses memory equal to the size of -.Ao Ar oldfile Ac -plus the size of -.Ao Ar newfile Ac , -but can tolerate a very small working set without a dramatic loss -of performance. -.Sh SEE ALSO -.Xr bsdiff 1 -.Sh AUTHORS -.An Colin Percival Aq cperciva@freebsd.org diff --git a/src/bsdiff-4.3/bspatch.c b/src/bsdiff-4.3/bspatch.c deleted file mode 100644 index f9d33ddd64..0000000000 --- a/src/bsdiff-4.3/bspatch.c +++ /dev/null @@ -1,224 +0,0 @@ -/*- - * Copyright 2003-2005 Colin Percival - * All rights reserved - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted providing that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING - * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#if 0 -__FBSDID("$FreeBSD: src/usr.bin/bsdiff/bspatch/bspatch.c,v 1.1 2005/08/06 01:59:06 cperciva Exp $"); -#endif - -#include <bzlib.h> -#include <stdlib.h> -#include <stdio.h> -#include <string.h> -#include <err.h> -#include <errno.h> -#include <unistd.h> -#include <fcntl.h> -#include <sys/types.h> - -static off_t offtin(u_char *buf) -{ - off_t y; - - y=buf[7]&0x7F; - y=y*256;y+=buf[6]; - y=y*256;y+=buf[5]; - y=y*256;y+=buf[4]; - y=y*256;y+=buf[3]; - y=y*256;y+=buf[2]; - y=y*256;y+=buf[1]; - y=y*256;y+=buf[0]; - - if(buf[7]&0x80) y=-y; - - return y; -} - - -void writeFull(const char * name, int fd, - const unsigned char * buf, size_t count) -{ - while (count) { - ssize_t res = write(fd, (char *) buf, count); - if (res == -1) { - if (errno == EINTR) continue; - err(1,"writing to %s",name); - } - count -= res; - buf += res; - } -} - - -int main(int argc,char * argv[]) -{ - FILE * f, * cpf, * dpf, * epf; - BZFILE * cpfbz2, * dpfbz2, * epfbz2; - int cbz2err, dbz2err, ebz2err; - int fd; - ssize_t oldsize,newsize; - ssize_t bzctrllen,bzdatalen; - u_char header[32],buf[8]; - u_char *old, *new; - off_t oldpos,newpos; - off_t ctrl[3]; - off_t lenread; - off_t i; - - if(argc!=4) errx(1,"usage: %s oldfile newfile patchfile\n",argv[0]); - - /* Open patch file */ - if ((f = fopen(argv[3], "r")) == NULL) - err(1, "fopen(%s)", argv[3]); - - /* - File format: - 0 8 "BSDIFF40" - 8 8 X - 16 8 Y - 24 8 sizeof(newfile) - 32 X bzip2(control block) - 32+X Y bzip2(diff block) - 32+X+Y ??? bzip2(extra block) - with control block a set of triples (x,y,z) meaning "add x bytes - from oldfile to x bytes from the diff block; copy y bytes from the - extra block; seek forwards in oldfile by z bytes". - */ - - /* Read header */ - if (fread(header, 1, 32, f) < 32) { - if (feof(f)) - errx(1, "Corrupt patch\n"); - err(1, "fread(%s)", argv[3]); - } - - /* Check for appropriate magic */ - if (memcmp(header, "BSDIFF40", 8) != 0) - errx(1, "Corrupt patch\n"); - - /* Read lengths from header */ - bzctrllen=offtin(header+8); - bzdatalen=offtin(header+16); - newsize=offtin(header+24); - if((bzctrllen<0) || (bzdatalen<0) || (newsize<0)) - errx(1,"Corrupt patch\n"); - - /* Close patch file and re-open it via libbzip2 at the right places */ - if (fclose(f)) - err(1, "fclose(%s)", argv[3]); - if ((cpf = fopen(argv[3], "r")) == NULL) - err(1, "fopen(%s)", argv[3]); - if (fseeko(cpf, 32, SEEK_SET)) - err(1, "fseeko(%s, %lld)", argv[3], - (long long)32); - if ((cpfbz2 = BZ2_bzReadOpen(&cbz2err, cpf, 0, 0, NULL, 0)) == NULL) - errx(1, "BZ2_bzReadOpen, bz2err = %d", cbz2err); - if ((dpf = fopen(argv[3], "r")) == NULL) - err(1, "fopen(%s)", argv[3]); - if (fseeko(dpf, 32 + bzctrllen, SEEK_SET)) - err(1, "fseeko(%s, %lld)", argv[3], - (long long)(32 + bzctrllen)); - if ((dpfbz2 = BZ2_bzReadOpen(&dbz2err, dpf, 0, 0, NULL, 0)) == NULL) - errx(1, "BZ2_bzReadOpen, bz2err = %d", dbz2err); - if ((epf = fopen(argv[3], "r")) == NULL) - err(1, "fopen(%s)", argv[3]); - if (fseeko(epf, 32 + bzctrllen + bzdatalen, SEEK_SET)) - err(1, "fseeko(%s, %lld)", argv[3], - (long long)(32 + bzctrllen + bzdatalen)); - if ((epfbz2 = BZ2_bzReadOpen(&ebz2err, epf, 0, 0, NULL, 0)) == NULL) - errx(1, "BZ2_bzReadOpen, bz2err = %d", ebz2err); - - if(((fd=open(argv[1],O_RDONLY,0))<0) || - ((oldsize=lseek(fd,0,SEEK_END))==-1) || - ((old=malloc(oldsize+1))==NULL) || - (lseek(fd,0,SEEK_SET)!=0) || - (read(fd,old,oldsize)!=oldsize) || - (close(fd)==-1)) err(1,"%s",argv[1]); - if((new=malloc(newsize+1))==NULL) err(1,NULL); - - oldpos=0;newpos=0; - while(newpos<newsize) { - /* Read control data */ - for(i=0;i<=2;i++) { - lenread = BZ2_bzRead(&cbz2err, cpfbz2, buf, 8); - if ((lenread < 8) || ((cbz2err != BZ_OK) && - (cbz2err != BZ_STREAM_END))) - errx(1, "Corrupt patch\n"); - ctrl[i]=offtin(buf); - }; - - /* Sanity-check */ - if(newpos+ctrl[0]>newsize) - errx(1,"Corrupt patch\n"); - - /* Read diff string */ - lenread = BZ2_bzRead(&dbz2err, dpfbz2, new + newpos, ctrl[0]); - if ((lenread < ctrl[0]) || - ((dbz2err != BZ_OK) && (dbz2err != BZ_STREAM_END))) - errx(1, "Corrupt patch\n"); - - /* Add old data to diff string */ - for(i=0;i<ctrl[0];i++) - if((oldpos+i>=0) && (oldpos+i<oldsize)) - new[newpos+i]+=old[oldpos+i]; - - /* Adjust pointers */ - newpos+=ctrl[0]; - oldpos+=ctrl[0]; - - /* Sanity-check */ - if(newpos+ctrl[1]>newsize) - errx(1,"Corrupt patch\n"); - - /* Read extra string */ - lenread = BZ2_bzRead(&ebz2err, epfbz2, new + newpos, ctrl[1]); - if ((lenread < ctrl[1]) || - ((ebz2err != BZ_OK) && (ebz2err != BZ_STREAM_END))) - errx(1, "Corrupt patch\n"); - - /* Adjust pointers */ - newpos+=ctrl[1]; - oldpos+=ctrl[2]; - }; - - /* Clean up the bzip2 reads */ - BZ2_bzReadClose(&cbz2err, cpfbz2); - BZ2_bzReadClose(&dbz2err, dpfbz2); - BZ2_bzReadClose(&ebz2err, epfbz2); - if (fclose(cpf) || fclose(dpf) || fclose(epf)) - err(1, "fclose(%s)", argv[3]); - - /* Write the new file */ - if((fd=open(argv[2],O_CREAT|O_TRUNC|O_WRONLY,0666))<0) - err(1,"%s",argv[2]); - writeFull(argv[2], fd, new, newsize); - if(close(fd)==-1) - err(1,"%s",argv[2]); - - free(new); - free(old); - - return 0; -} diff --git a/src/bsdiff-4.3/compat-include/err.h b/src/bsdiff-4.3/compat-include/err.h deleted file mode 100644 index a851ded6f9..0000000000 --- a/src/bsdiff-4.3/compat-include/err.h +++ /dev/null @@ -1,12 +0,0 @@ -/* Simulate BSD's <err.h> functionality. */ - -#ifndef COMPAT_ERR_H_INCLUDED -#define COMPAT_ERR_H_INCLUDED 1 - -#include <stdio.h> -#include <stdlib.h> - -#define err(rc,...) do { fprintf(stderr,__VA_ARGS__); exit(rc); } while(0) -#define errx(rc,...) do { fprintf(stderr,__VA_ARGS__); exit(rc); } while(0) - -#endif diff --git a/src/bsdiff-4.3/local.mk b/src/bsdiff-4.3/local.mk deleted file mode 100644 index c957ceab0c..0000000000 --- a/src/bsdiff-4.3/local.mk +++ /dev/null @@ -1,11 +0,0 @@ -programs += bsdiff bspatch - -bsdiff_DIR := $(d) -bsdiff_SOURCES := $(d)/bsdiff.c -bsdiff_LDFLAGS = -lbz2 $(bsddiff_compat_include) -bsdiff_INSTALL_DIR = $(libexecdir)/nix - -bspatch_DIR := $(d) -bspatch_SOURCES := $(d)/bspatch.c -bspatch_LDFLAGS = -lbz2 $(bsddiff_compat_include) -bspatch_INSTALL_DIR = $(libexecdir)/nix diff --git a/src/download-via-ssh/download-via-ssh.cc b/src/download-via-ssh/download-via-ssh.cc deleted file mode 100644 index 6361e71e99..0000000000 --- a/src/download-via-ssh/download-via-ssh.cc +++ /dev/null @@ -1,155 +0,0 @@ -#include "shared.hh" -#include "util.hh" -#include "serialise.hh" -#include "archive.hh" -#include "affinity.hh" -#include "globals.hh" -#include "serve-protocol.hh" -#include "worker-protocol.hh" -#include "store-api.hh" - -#include <iostream> -#include <unistd.h> - -using namespace nix; - -// !!! TODO: -// * Respect more than the first host -// * use a database -// * show progress - - -static std::pair<FdSink, FdSource> connect(const string & conn) -{ - Pipe to, from; - to.create(); - from.create(); - pid_t child = fork(); - switch (child) { - case -1: - throw SysError("unable to fork"); - case 0: - try { - restoreAffinity(); - if (dup2(to.readSide, STDIN_FILENO) == -1) - throw SysError("dupping stdin"); - if (dup2(from.writeSide, STDOUT_FILENO) == -1) - throw SysError("dupping stdout"); - execlp("ssh" - , "ssh" - , "-x" - , "-T" - , conn.c_str() - , "nix-store --serve" - , NULL); - throw SysError("executing ssh"); - } catch (std::exception & e) { - std::cerr << "error: " << e.what() << std::endl; - } - _exit(1); - } - // If child exits unexpectedly, we'll EPIPE or EOF early. - // If we exit unexpectedly, child will EPIPE or EOF early. - // So no need to keep track of it. - - return std::pair<FdSink, FdSource>(to.writeSide.borrow(), from.readSide.borrow()); -} - - -static void substitute(std::pair<FdSink, FdSource> & pipes, Path storePath, Path destPath) -{ - writeInt(cmdSubstitute, pipes.first); - writeString(storePath, pipes.first); - pipes.first.flush(); - restorePath(destPath, pipes.second); - std::cout << std::endl; -} - - -static void query(std::pair<FdSink, FdSource> & pipes) -{ - writeInt(cmdQuery, pipes.first); - for (string line; getline(std::cin, line);) { - Strings tokenized = tokenizeString<Strings>(line); - string cmd = tokenized.front(); - tokenized.pop_front(); - if (cmd == "have") { - writeInt(qCmdHave, pipes.first); - writeStrings(tokenized, pipes.first); - pipes.first.flush(); - PathSet paths = readStrings<PathSet>(pipes.second); - foreach (PathSet::iterator, i, paths) - std::cout << *i << std::endl; - } else if (cmd == "info") { - writeInt(qCmdInfo, pipes.first); - writeStrings(tokenized, pipes.first); - pipes.first.flush(); - while (1) { - Path path = readString(pipes.second); - if (path.empty()) break; - assertStorePath(path); - std::cout << path << std::endl; - string deriver = readString(pipes.second); - if (!deriver.empty()) assertStorePath(deriver); - std::cout << deriver << std::endl; - PathSet references = readStorePaths<PathSet>(pipes.second); - std::cout << references.size() << std::endl; - foreach (PathSet::iterator, i, references) - std::cout << *i << std::endl; - std::cout << readLongLong(pipes.second) << std::endl; - std::cout << readLongLong(pipes.second) << std::endl; - } - } else - throw Error(format("unknown substituter query `%1%'") % cmd); - std::cout << std::endl; - } -} - - -void run(Strings args) -{ - if (args.empty()) - throw UsageError("download-via-ssh requires an argument"); - - if (settings.sshSubstituterHosts.empty()) - return; - - std::cout << std::endl; - - string host = settings.sshSubstituterHosts.front(); - std::pair<FdSink, FdSource> pipes = connect(host); - - /* Exchange the greeting */ - writeInt(SERVE_MAGIC_1, pipes.first); - pipes.first.flush(); - unsigned int magic = readInt(pipes.second); - if (magic != SERVE_MAGIC_2) - throw Error("protocol mismatch"); - readInt(pipes.second); // Server version, unused for now - writeInt(SERVE_PROTOCOL_VERSION, pipes.first); - pipes.first.flush(); - - Strings::iterator i = args.begin(); - if (*i == "--query") - query(pipes); - else if (*i == "--substitute") - if (args.size() != 3) - throw UsageError("download-via-ssh: --substitute takes exactly two arguments"); - else { - Path storePath = *++i; - Path destPath = *++i; - printMsg(lvlError, format("downloading `%1%' via SSH from `%2%'...") % storePath % host); - substitute(pipes, storePath, destPath); - } - else - throw UsageError(format("download-via-ssh: unknown command `%1%'") % *i); -} - - -void printHelp() -{ - std::cerr << "Usage: download-via-ssh --query|--substitute store-path dest-path" << std::endl; -} - - -string programId = "download-via-ssh"; diff --git a/src/download-via-ssh/local.mk b/src/download-via-ssh/local.mk deleted file mode 100644 index 80f4c385ac..0000000000 --- a/src/download-via-ssh/local.mk +++ /dev/null @@ -1,11 +0,0 @@ -programs += download-via-ssh - -download-via-ssh_DIR := $(d) - -download-via-ssh_SOURCES := $(d)/download-via-ssh.cc - -download-via-ssh_INSTALL_DIR := $(libexecdir)/nix/substituters - -download-via-ssh_CXXFLAGS = -Isrc/nix-store - -download-via-ssh_LIBS = libmain libstore libutil libformat diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc deleted file mode 100644 index 4f28a549f0..0000000000 --- a/src/libexpr/attr-path.cc +++ /dev/null @@ -1,97 +0,0 @@ -#include "attr-path.hh" -#include "eval-inline.hh" -#include "util.hh" - - -namespace nix { - - -static Strings parseAttrPath(const string & s) -{ - Strings res; - string cur; - string::const_iterator i = s.begin(); - while (i != s.end()) { - if (*i == '.') { - res.push_back(cur); - cur.clear(); - } else if (*i == '"') { - ++i; - while (1) { - if (i == s.end()) - throw Error(format("missing closing quote in selection path `%1%'") % s); - if (*i == '"') break; - cur.push_back(*i++); - } - } else - cur.push_back(*i); - ++i; - } - if (!cur.empty()) res.push_back(cur); - return res; -} - - -Value * findAlongAttrPath(EvalState & state, const string & attrPath, - Bindings & autoArgs, Value & vIn) -{ - Strings tokens = parseAttrPath(attrPath); - - Error attrError = - Error(format("attribute selection path `%1%' does not match expression") % attrPath); - - Value * v = &vIn; - - foreach (Strings::iterator, i, tokens) { - - /* Is *i an index (integer) or a normal attribute name? */ - enum { apAttr, apIndex } apType = apAttr; - string attr = *i; - unsigned int attrIndex; - if (string2Int(attr, attrIndex)) apType = apIndex; - - /* Evaluate the expression. */ - Value * vNew = state.allocValue(); - state.autoCallFunction(autoArgs, *v, *vNew); - v = vNew; - state.forceValue(*v); - - /* It should evaluate to either a set or an expression, - according to what is specified in the attrPath. */ - - if (apType == apAttr) { - - if (v->type != tAttrs) - throw TypeError( - format("the expression selected by the selection path `%1%' should be a set but is %2%") - % attrPath % showType(*v)); - - if (attr.empty()) - throw Error(format("empty attribute name in selection path `%1%'") % attrPath); - - Bindings::iterator a = v->attrs->find(state.symbols.create(attr)); - if (a == v->attrs->end()) - throw Error(format("attribute `%1%' in selection path `%2%' not found") % attr % attrPath); - v = &*a->value; - } - - else if (apType == apIndex) { - - if (v->type != tList) - throw TypeError( - format("the expression selected by the selection path `%1%' should be a list but is %2%") - % attrPath % showType(*v)); - - if (attrIndex >= v->list.length) - throw Error(format("list index %1% in selection path `%2%' is out of range") % attrIndex % attrPath); - - v = v->list.elems[attrIndex]; - } - - } - - return v; -} - - -} diff --git a/src/libexpr/attr-path.hh b/src/libexpr/attr-path.hh deleted file mode 100644 index 46a3419509..0000000000 --- a/src/libexpr/attr-path.hh +++ /dev/null @@ -1,13 +0,0 @@ -#pragma once - -#include "eval.hh" - -#include <string> -#include <map> - -namespace nix { - -Value * findAlongAttrPath(EvalState & state, const string & attrPath, - Bindings & autoArgs, Value & vIn); - -} diff --git a/src/libexpr/common-opts.cc b/src/libexpr/common-opts.cc deleted file mode 100644 index 9b3421f6c4..0000000000 --- a/src/libexpr/common-opts.cc +++ /dev/null @@ -1,59 +0,0 @@ -#include "common-opts.hh" -#include "../libmain/shared.hh" -#include "util.hh" - - -namespace nix { - - -bool parseOptionArg(const string & arg, Strings::iterator & i, - const Strings::iterator & argsEnd, EvalState & state, - Bindings & autoArgs) -{ - if (arg != "--arg" && arg != "--argstr") return false; - - UsageError error(format("`%1%' requires two arguments") % arg); - - if (i == argsEnd) throw error; - string name = *i++; - if (i == argsEnd) throw error; - string value = *i++; - - /* !!! check for duplicates! */ - Value * v = state.allocValue(); - autoArgs.push_back(Attr(state.symbols.create(name), v)); - - if (arg == "--arg") - state.mkThunk_(*v, state.parseExprFromString(value, absPath("."))); - else - mkString(*v, value); - - autoArgs.sort(); // !!! inefficient - - return true; -} - - -bool parseSearchPathArg(const string & arg, Strings::iterator & i, - const Strings::iterator & argsEnd, EvalState & state) -{ - if (arg != "-I") return false; - if (i == argsEnd) throw UsageError(format("`%1%' requires an argument") % arg);; - state.addToSearchPath(*i++, true); - return true; -} - - -Path lookupFileArg(EvalState & state, string s) -{ - if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { - Path p = s.substr(1, s.size() - 2); - Path p2 = state.findFile(p); - if (p2 == "") throw Error(format("file `%1%' was not found in the Nix search path (add it using $NIX_PATH or -I)") % p); - return p2; - } else - return absPath(s); -} - - -} diff --git a/src/libexpr/common-opts.hh b/src/libexpr/common-opts.hh deleted file mode 100644 index e2e3fe7717..0000000000 --- a/src/libexpr/common-opts.hh +++ /dev/null @@ -1,17 +0,0 @@ -#pragma once - -#include "eval.hh" - -namespace nix { - -/* Some common option parsing between nix-env and nix-instantiate. */ -bool parseOptionArg(const string & arg, Strings::iterator & i, - const Strings::iterator & argsEnd, EvalState & state, - Bindings & autoArgs); - -bool parseSearchPathArg(const string & arg, Strings::iterator & i, - const Strings::iterator & argsEnd, EvalState & state); - -Path lookupFileArg(EvalState & state, string s); - -} diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh deleted file mode 100644 index 5801a20c3b..0000000000 --- a/src/libexpr/eval-inline.hh +++ /dev/null @@ -1,59 +0,0 @@ -#pragma once - -#include "eval.hh" - -#define LocalNoInline(f) static f __attribute__((noinline)); f -#define LocalNoInlineNoReturn(f) static f __attribute__((noinline, noreturn)); f - -namespace nix { - -LocalNoInlineNoReturn(void throwEvalError(const char * s)) -{ - throw EvalError(s); -} - -LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v)) -{ - throw TypeError(format(s) % showType(v)); -} - - -void EvalState::forceValue(Value & v) -{ - if (v.type == tThunk) { - Env * env = v.thunk.env; - Expr * expr = v.thunk.expr; - try { - v.type = tBlackhole; - //checkInterrupt(); - expr->eval(*this, *env, v); - } catch (Error & e) { - v.type = tThunk; - v.thunk.env = env; - v.thunk.expr = expr; - throw; - } - } - else if (v.type == tApp) - callFunction(*v.app.left, *v.app.right, v); - else if (v.type == tBlackhole) - throwEvalError("infinite recursion encountered"); -} - - -inline void EvalState::forceAttrs(Value & v) -{ - forceValue(v); - if (v.type != tAttrs) - throwTypeError("value is %1% while a set was expected", v); -} - - -inline void EvalState::forceList(Value & v) -{ - forceValue(v); - if (v.type != tList) - throwTypeError("value is %1% while a list was expected", v); -} - -} diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc deleted file mode 100644 index bbf494387e..0000000000 --- a/src/libexpr/eval.cc +++ /dev/null @@ -1,1438 +0,0 @@ -#include "eval.hh" -#include "hash.hh" -#include "util.hh" -#include "store-api.hh" -#include "derivations.hh" -#include "globals.hh" -#include "eval-inline.hh" - -#include <algorithm> -#include <cstring> -#include <unistd.h> -#include <sys/time.h> -#include <sys/resource.h> - -#if HAVE_BOEHMGC - -#include <gc/gc.h> -#include <gc/gc_cpp.h> - -#define NEW new (UseGC) - -#else - -#define GC_STRDUP strdup -#define GC_MALLOC malloc - -#define NEW new - -#endif - - -namespace nix { - - -Bindings::iterator Bindings::find(const Symbol & name) -{ - Attr key(name, 0); - iterator i = lower_bound(begin(), end(), key); - if (i != end() && i->name == name) return i; - return end(); -} - - -void Bindings::sort() -{ - std::sort(begin(), end()); -} - - -std::ostream & operator << (std::ostream & str, const Value & v) -{ - switch (v.type) { - case tInt: - str << v.integer; - break; - case tBool: - str << (v.boolean ? "true" : "false"); - break; - case tString: - str << "\""; - for (const char * i = v.string.s; *i; i++) - if (*i == '\"' || *i == '\\') str << "\\" << *i; - else if (*i == '\n') str << "\\n"; - else if (*i == '\r') str << "\\r"; - else if (*i == '\t') str << "\\t"; - else str << *i; - str << "\""; - break; - case tPath: - str << v.path; // !!! escaping? - break; - case tNull: - str << "null"; - break; - case tAttrs: { - str << "{ "; - typedef std::map<string, Value *> Sorted; - Sorted sorted; - foreach (Bindings::iterator, i, *v.attrs) - sorted[i->name] = i->value; - foreach (Sorted::iterator, i, sorted) - str << i->first << " = " << *i->second << "; "; - str << "}"; - break; - } - case tList: - str << "[ "; - for (unsigned int n = 0; n < v.list.length; ++n) - str << *v.list.elems[n] << " "; - str << "]"; - break; - case tThunk: - case tApp: - str << "<CODE>"; - break; - case tLambda: - str << "<LAMBDA>"; - break; - case tPrimOp: - str << "<PRIMOP>"; - break; - case tPrimOpApp: - str << "<PRIMOP-APP>"; - break; - default: - throw Error("invalid value"); - } - return str; -} - - -string showType(const Value & v) -{ - switch (v.type) { - case tInt: return "an integer"; - case tBool: return "a boolean"; - case tString: return "a string"; - case tPath: return "a path"; - case tNull: return "null"; - case tAttrs: return "a set"; - case tList: return "a list"; - case tThunk: return "a thunk"; - case tApp: return "a function application"; - case tLambda: return "a function"; - case tBlackhole: return "a black hole"; - case tPrimOp: return "a built-in function"; - case tPrimOpApp: return "a partially applied built-in function"; - } - abort(); -} - - -/* Called when the Boehm GC runs out of memory. */ -static void * oomHandler(size_t requested) -{ - /* Convert this to a proper C++ exception. */ - throw std::bad_alloc(); -} - - -static Symbol getName(const AttrName & name, EvalState & state, Env & env) { - if (name.symbol.set()) { - return name.symbol; - } else { - Value nameValue; - name.expr->eval(state, env, nameValue); - state.forceStringNoCtx(nameValue); - return state.symbols.create(nameValue.string.s); - } -} - - -EvalState::EvalState() - : sWith(symbols.create("<with>")) - , sOutPath(symbols.create("outPath")) - , sDrvPath(symbols.create("drvPath")) - , sType(symbols.create("type")) - , sMeta(symbols.create("meta")) - , sName(symbols.create("name")) - , sValue(symbols.create("value")) - , sSystem(symbols.create("system")) - , sOverrides(symbols.create("__overrides")) - , sOutputs(symbols.create("outputs")) - , sOutputName(symbols.create("outputName")) - , sIgnoreNulls(symbols.create("__ignoreNulls")) - , sFile(symbols.create("file")) - , sLine(symbols.create("line")) - , sColumn(symbols.create("column")) - , repair(false) - , baseEnv(allocEnv(128)) - , staticBaseEnv(false, 0) - , baseEnvDispl(0) -{ - nrEnvs = nrValuesInEnvs = nrValues = nrListElems = 0; - nrAttrsets = nrOpUpdates = nrOpUpdateValuesCopied = 0; - nrListConcats = nrPrimOpCalls = nrFunctionCalls = 0; - countCalls = getEnv("NIX_COUNT_CALLS", "0") != "0"; - -#if HAVE_BOEHMGC - static bool gcInitialised = false; - if (!gcInitialised) { - - /* Initialise the Boehm garbage collector. This isn't - necessary on most platforms, but for portability we do it - anyway. */ - GC_INIT(); - - GC_oom_fn = oomHandler; - - /* Set the initial heap size to something fairly big (25% of - physical RAM, up to a maximum of 384 MiB) so that in most - cases we don't need to garbage collect at all. (Collection - has a fairly significant overhead.) The heap size can be - overridden through libgc's GC_INITIAL_HEAP_SIZE environment - variable. We should probably also provide a nix.conf - setting for this. Note that GC_expand_hp() causes a lot of - virtual, but not physical (resident) memory to be - allocated. This might be a problem on systems that don't - overcommit. */ - if (!getenv("GC_INITIAL_HEAP_SIZE")) { - size_t maxSize = 384 * 1024 * 1024; - size_t size = 32 * 1024 * 1024; -#if HAVE_SYSCONF && defined(_SC_PAGESIZE) && defined(_SC_PHYS_PAGES) - long pageSize = sysconf(_SC_PAGESIZE); - long pages = sysconf(_SC_PHYS_PAGES); - if (pageSize != -1) - size = (pageSize * pages) / 4; // 25% of RAM - if (size > maxSize) size = maxSize; -#endif - debug(format("setting initial heap size to %1% bytes") % size); - GC_expand_hp(size); - } - - gcInitialised = true; - } -#endif - - /* Initialise the Nix expression search path. */ - searchPathInsertionPoint = searchPath.end(); - Strings paths = tokenizeString<Strings>(getEnv("NIX_PATH", ""), ":"); - foreach (Strings::iterator, i, paths) addToSearchPath(*i); - addToSearchPath("nix=" + settings.nixDataDir + "/nix/corepkgs"); - searchPathInsertionPoint = searchPath.begin(); - - createBaseEnv(); -} - - -EvalState::~EvalState() -{ -} - - -void EvalState::addConstant(const string & name, Value & v) -{ - Value * v2 = allocValue(); - *v2 = v; - staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl; - baseEnv.values[baseEnvDispl++] = v2; - string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name; - baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v2)); -} - - -void EvalState::addPrimOp(const string & name, - unsigned int arity, PrimOpFun primOp) -{ - Value * v = allocValue(); - string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name; - Symbol sym = symbols.create(name2); - v->type = tPrimOp; - v->primOp = NEW PrimOp(primOp, arity, sym); - staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl; - baseEnv.values[baseEnvDispl++] = v; - baseEnv.values[0]->attrs->push_back(Attr(sym, v)); -} - - -void EvalState::getBuiltin(const string & name, Value & v) -{ - v = *baseEnv.values[0]->attrs->find(symbols.create(name))->value; -} - - -/* Every "format" object (even temporary) takes up a few hundred bytes - of stack space, which is a real killer in the recursive - evaluator. So here are some helper functions for throwing - exceptions. */ - -LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2)) -{ - throw EvalError(format(s) % s2); -} - -LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const string & s3)) -{ - throw EvalError(format(s) % s2 % s3); -} - -LocalNoInlineNoReturn(void throwEvalError(const char * s, const Symbol & sym, const Pos & p1, const Pos & p2)) -{ - throw EvalError(format(s) % sym % p1 % p2); -} - -LocalNoInlineNoReturn(void throwTypeError(const char * s)) -{ - throw TypeError(s); -} - -LocalNoInlineNoReturn(void throwTypeError(const char * s, const string & s1)) -{ - throw TypeError(format(s) % s1); -} - -LocalNoInlineNoReturn(void throwTypeError(const char * s, const string & s1, const string & s2)) -{ - throw TypeError(format(s) % s1 % s2); -} - -LocalNoInlineNoReturn(void throwTypeError(const char * s, const ExprLambda & fun, const Symbol & s2)) -{ - throw TypeError(format(s) % fun.showNamePos() % s2); -} - -LocalNoInlineNoReturn(void throwAssertionError(const char * s, const Pos & pos)) -{ - throw AssertionError(format(s) % pos); -} - -LocalNoInlineNoReturn(void throwUndefinedVarError(const char * s, const string & s1, const Pos & pos)) -{ - throw UndefinedVarError(format(s) % s1 % pos); -} - -LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2)) -{ - e.addPrefix(format(s) % s2); -} - -LocalNoInline(void addErrorPrefix(Error & e, const char * s, const ExprLambda & fun)) -{ - e.addPrefix(format(s) % fun.showNamePos()); -} - -LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2, const Pos & pos)) -{ - e.addPrefix(format(s) % s2 % pos); -} - - -void mkString(Value & v, const char * s) -{ - mkStringNoCopy(v, GC_STRDUP(s)); -} - - -void mkString(Value & v, const string & s, const PathSet & context) -{ - mkString(v, s.c_str()); - if (!context.empty()) { - unsigned int n = 0; - v.string.context = (const char * *) - GC_MALLOC((context.size() + 1) * sizeof(char *)); - foreach (PathSet::const_iterator, i, context) - v.string.context[n++] = GC_STRDUP(i->c_str()); - v.string.context[n] = 0; - } -} - - -void mkPath(Value & v, const char * s) -{ - mkPathNoCopy(v, GC_STRDUP(s)); -} - - -inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval) -{ - for (unsigned int l = var.level; l; --l, env = env->up) ; - - if (!var.fromWith) return env->values[var.displ]; - - while (1) { - if (!env->haveWithAttrs) { - if (noEval) return 0; - Value * v = allocValue(); - evalAttrs(*env->up, (Expr *) env->values[0], *v); - env->values[0] = v; - env->haveWithAttrs = true; - } - Bindings::iterator j = env->values[0]->attrs->find(var.name); - if (j != env->values[0]->attrs->end()) { - if (countCalls && j->pos) attrSelects[*j->pos]++; - return j->value; - } - if (!env->prevWith) - throwUndefinedVarError("undefined variable `%1%' at %2%", var.name, var.pos); - for (unsigned int l = env->prevWith; l; --l, env = env->up) ; - } -} - - -Value * EvalState::allocValue() -{ - nrValues++; - return (Value *) GC_MALLOC(sizeof(Value)); -} - - -Env & EvalState::allocEnv(unsigned int size) -{ - nrEnvs++; - nrValuesInEnvs += size; - Env * env = (Env *) GC_MALLOC(sizeof(Env) + size * sizeof(Value *)); - - /* Clear the values because maybeThunk() and lookupVar fromWith expects this. */ - for (unsigned i = 0; i < size; ++i) - env->values[i] = 0; - - return *env; -} - - -Value * EvalState::allocAttr(Value & vAttrs, const Symbol & name) -{ - Value * v = allocValue(); - vAttrs.attrs->push_back(Attr(name, v)); - return v; -} - - -void EvalState::mkList(Value & v, unsigned int length) -{ - v.type = tList; - v.list.length = length; - v.list.elems = length ? (Value * *) GC_MALLOC(length * sizeof(Value *)) : 0; - nrListElems += length; -} - - -void EvalState::mkAttrs(Value & v, unsigned int expected) -{ - clearValue(v); - v.type = tAttrs; - v.attrs = NEW Bindings; - v.attrs->reserve(expected); - nrAttrsets++; -} - - -unsigned long nrThunks = 0; - -static inline void mkThunk(Value & v, Env & env, Expr * expr) -{ - v.type = tThunk; - v.thunk.env = &env; - v.thunk.expr = expr; - nrThunks++; -} - - -void EvalState::mkThunk_(Value & v, Expr * expr) -{ - mkThunk(v, baseEnv, expr); -} - - -void EvalState::mkPos(Value & v, Pos * pos) -{ - if (pos) { - mkAttrs(v, 3); - mkString(*allocAttr(v, sFile), pos->file); - mkInt(*allocAttr(v, sLine), pos->line); - mkInt(*allocAttr(v, sColumn), pos->column); - v.attrs->sort(); - } else - mkNull(v); -} - - -/* Create a thunk for the delayed computation of the given expression - in the given environment. But if the expression is a variable, - then look it up right away. This significantly reduces the number - of thunks allocated. */ -Value * Expr::maybeThunk(EvalState & state, Env & env) -{ - Value * v = state.allocValue(); - mkThunk(*v, env, this); - return v; -} - - -unsigned long nrAvoided = 0; - -Value * ExprVar::maybeThunk(EvalState & state, Env & env) -{ - Value * v = state.lookupVar(&env, *this, true); - /* The value might not be initialised in the environment yet. - In that case, ignore it. */ - if (v) { nrAvoided++; return v; } - return Expr::maybeThunk(state, env); -} - - -Value * ExprString::maybeThunk(EvalState & state, Env & env) -{ - nrAvoided++; - return &v; -} - -Value * ExprInt::maybeThunk(EvalState & state, Env & env) -{ - nrAvoided++; - return &v; -} - -Value * ExprPath::maybeThunk(EvalState & state, Env & env) -{ - nrAvoided++; - return &v; -} - - -void EvalState::evalFile(const Path & path, Value & v) -{ - FileEvalCache::iterator i; - if ((i = fileEvalCache.find(path)) != fileEvalCache.end()) { - v = i->second; - return; - } - - Path path2 = resolveExprPath(path); - if ((i = fileEvalCache.find(path2)) != fileEvalCache.end()) { - v = i->second; - return; - } - - startNest(nest, lvlTalkative, format("evaluating file `%1%'") % path2); - Expr * e = parseExprFromFile(path2); - try { - eval(e, v); - } catch (Error & e) { - addErrorPrefix(e, "while evaluating the file `%1%':\n", path2); - throw; - } - - fileEvalCache[path2] = v; - if (path != path2) fileEvalCache[path] = v; -} - - -void EvalState::resetFileCache() -{ - fileEvalCache.clear(); -} - - -void EvalState::eval(Expr * e, Value & v) -{ - e->eval(*this, baseEnv, v); -} - - -inline bool EvalState::evalBool(Env & env, Expr * e) -{ - Value v; - e->eval(*this, env, v); - if (v.type != tBool) - throwTypeError("value is %1% while a Boolean was expected", v); - return v.boolean; -} - - -inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v) -{ - e->eval(*this, env, v); - if (v.type != tAttrs) - throwTypeError("value is %1% while a set was expected", v); -} - - -void Expr::eval(EvalState & state, Env & env, Value & v) -{ - abort(); -} - - -void ExprInt::eval(EvalState & state, Env & env, Value & v) -{ - v = this->v; -} - - -void ExprString::eval(EvalState & state, Env & env, Value & v) -{ - v = this->v; -} - - -void ExprPath::eval(EvalState & state, Env & env, Value & v) -{ - v = this->v; -} - - -void ExprAttrs::eval(EvalState & state, Env & env, Value & v) -{ - state.mkAttrs(v, attrs.size()); - Env *dynamicEnv = &env; - - if (recursive) { - /* Create a new environment that contains the attributes in - this `rec'. */ - Env & env2(state.allocEnv(attrs.size())); - env2.up = &env; - dynamicEnv = &env2; - - AttrDefs::iterator overrides = attrs.find(state.sOverrides); - bool hasOverrides = overrides != attrs.end(); - - /* The recursive attributes are evaluated in the new - environment, while the inherited attributes are evaluated - in the original environment. */ - unsigned int displ = 0; - foreach (AttrDefs::iterator, i, attrs) { - Value * vAttr; - if (hasOverrides && !i->second.inherited) { - vAttr = state.allocValue(); - mkThunk(*vAttr, env2, i->second.e); - } else - vAttr = i->second.e->maybeThunk(state, i->second.inherited ? env : env2); - env2.values[displ++] = vAttr; - v.attrs->push_back(Attr(i->first, vAttr, &i->second.pos)); - } - - /* If the rec contains an attribute called `__overrides', then - evaluate it, and add the attributes in that set to the rec. - This allows overriding of recursive attributes, which is - otherwise not possible. (You can use the // operator to - replace an attribute, but other attributes in the rec will - still reference the original value, because that value has - been substituted into the bodies of the other attributes. - Hence we need __overrides.) */ - if (hasOverrides) { - Value * vOverrides = (*v.attrs)[overrides->second.displ].value; - state.forceAttrs(*vOverrides); - foreach (Bindings::iterator, i, *vOverrides->attrs) { - AttrDefs::iterator j = attrs.find(i->name); - if (j != attrs.end()) { - (*v.attrs)[j->second.displ] = *i; - env2.values[j->second.displ] = i->value; - } else - v.attrs->push_back(*i); - } - v.attrs->sort(); - } - } - - else - foreach (AttrDefs::iterator, i, attrs) - v.attrs->push_back(Attr(i->first, i->second.e->maybeThunk(state, env), &i->second.pos)); - - /* dynamic attrs apply *after* rec and __overrides */ - foreach (DynamicAttrDefs::iterator, i, dynamicAttrs) { - Value nameVal; - if (i->nameExpr->es->size() == 1) { - i->nameExpr->es->front()->eval(state, *dynamicEnv, nameVal); - state.forceValue(nameVal); - if (nameVal.type == tNull) - continue; - } - i->nameExpr->eval(state, *dynamicEnv, nameVal); - state.forceStringNoCtx(nameVal); - Symbol nameSym = state.symbols.create(nameVal.string.s); - Bindings::iterator j = v.attrs->find(nameSym); - if (j != v.attrs->end()) - throwEvalError("dynamic attribute `%1%' at %2% already defined at %3%", nameSym, i->pos, *j->pos); - - i->valueExpr->setName(nameSym); - /* Keep sorted order so find can catch duplicates */ - v.attrs->insert(lower_bound(v.attrs->begin(), v.attrs->end(), Attr(nameSym, 0)), - Attr(nameSym, i->valueExpr->maybeThunk(state, *dynamicEnv), &i->pos)); - } -} - - -void ExprLet::eval(EvalState & state, Env & env, Value & v) -{ - /* Create a new environment that contains the attributes in this - `let'. */ - Env & env2(state.allocEnv(attrs->attrs.size())); - env2.up = &env; - - /* The recursive attributes are evaluated in the new environment, - while the inherited attributes are evaluated in the original - environment. */ - unsigned int displ = 0; - foreach (ExprAttrs::AttrDefs::iterator, i, attrs->attrs) - env2.values[displ++] = i->second.e->maybeThunk(state, i->second.inherited ? env : env2); - - body->eval(state, env2, v); -} - - -void ExprList::eval(EvalState & state, Env & env, Value & v) -{ - state.mkList(v, elems.size()); - for (unsigned int n = 0; n < v.list.length; ++n) - v.list.elems[n] = elems[n]->maybeThunk(state, env); -} - - -void ExprVar::eval(EvalState & state, Env & env, Value & v) -{ - Value * v2 = state.lookupVar(&env, *this, false); - state.forceValue(*v2); - v = *v2; -} - - -unsigned long nrLookups = 0; - -void ExprSelect::eval(EvalState & state, Env & env, Value & v) -{ - Value vTmp; - Pos * pos = 0; - Value * vAttrs = &vTmp; - - e->eval(state, env, vTmp); - - try { - - foreach (AttrPath::const_iterator, i, attrPath) { - nrLookups++; - Bindings::iterator j; - Symbol name = getName(*i, state, env); - if (def) { - state.forceValue(*vAttrs); - if (vAttrs->type != tAttrs || - (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) - { - def->eval(state, env, v); - return; - } - } else { - state.forceAttrs(*vAttrs); - if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) - throwEvalError("attribute `%1%' missing", showAttrPath(attrPath)); - } - vAttrs = j->value; - pos = j->pos; - if (state.countCalls && pos) state.attrSelects[*pos]++; - } - - state.forceValue(*vAttrs); - - } catch (Error & e) { - if (pos && pos->file != state.sDerivationNix) - addErrorPrefix(e, "while evaluating the attribute `%1%' at %2%:\n", - showAttrPath(attrPath), *pos); - throw; - } - - v = *vAttrs; -} - - -void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v) -{ - Value vTmp; - Value * vAttrs = &vTmp; - - e->eval(state, env, vTmp); - - foreach (AttrPath::const_iterator, i, attrPath) { - state.forceValue(*vAttrs); - Bindings::iterator j; - Symbol name = getName(*i, state, env); - if (vAttrs->type != tAttrs || - (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) - { - mkBool(v, false); - return; - } else { - vAttrs = j->value; - } - } - - mkBool(v, true); -} - - -void ExprLambda::eval(EvalState & state, Env & env, Value & v) -{ - v.type = tLambda; - v.lambda.env = &env; - v.lambda.fun = this; -} - - -void ExprApp::eval(EvalState & state, Env & env, Value & v) -{ - /* FIXME: vFun prevents GCC from doing tail call optimisation. */ - Value vFun; - e1->eval(state, env, vFun); - state.callFunction(vFun, *(e2->maybeThunk(state, env)), v); -} - - -void EvalState::callPrimOp(Value & fun, Value & arg, Value & v) -{ - /* Figure out the number of arguments still needed. */ - unsigned int argsDone = 0; - Value * primOp = &fun; - while (primOp->type == tPrimOpApp) { - argsDone++; - primOp = primOp->primOpApp.left; - } - assert(primOp->type == tPrimOp); - unsigned int arity = primOp->primOp->arity; - unsigned int argsLeft = arity - argsDone; - - if (argsLeft == 1) { - /* We have all the arguments, so call the primop. */ - - /* Put all the arguments in an array. */ - Value * vArgs[arity]; - unsigned int n = arity - 1; - vArgs[n--] = &arg; - for (Value * arg = &fun; arg->type == tPrimOpApp; arg = arg->primOpApp.left) - vArgs[n--] = arg->primOpApp.right; - - /* And call the primop. */ - nrPrimOpCalls++; - if (countCalls) primOpCalls[primOp->primOp->name]++; - primOp->primOp->fun(*this, vArgs, v); - } else { - Value * fun2 = allocValue(); - *fun2 = fun; - v.type = tPrimOpApp; - v.primOpApp.left = fun2; - v.primOpApp.right = &arg; - } -} - - -void EvalState::callFunction(Value & fun, Value & arg, Value & v) -{ - if (fun.type == tPrimOp || fun.type == tPrimOpApp) { - callPrimOp(fun, arg, v); - return; - } - - if (fun.type != tLambda) - throwTypeError("attempt to call something which is not a function but %1%", fun); - - ExprLambda & lambda(*fun.lambda.fun); - - unsigned int size = - (lambda.arg.empty() ? 0 : 1) + - (lambda.matchAttrs ? lambda.formals->formals.size() : 0); - Env & env2(allocEnv(size)); - env2.up = fun.lambda.env; - - unsigned int displ = 0; - - if (!lambda.matchAttrs) - env2.values[displ++] = &arg; - - else { - forceAttrs(arg); - - if (!lambda.arg.empty()) - env2.values[displ++] = &arg; - - /* For each formal argument, get the actual argument. If - there is no matching actual argument but the formal - argument has a default, use the default. */ - unsigned int attrsUsed = 0; - foreach (Formals::Formals_::iterator, i, lambda.formals->formals) { - Bindings::iterator j = arg.attrs->find(i->name); - if (j == arg.attrs->end()) { - if (!i->def) throwTypeError("%1% called without required argument `%2%'", - lambda, i->name); - env2.values[displ++] = i->def->maybeThunk(*this, env2); - } else { - attrsUsed++; - env2.values[displ++] = j->value; - } - } - - /* Check that each actual argument is listed as a formal - argument (unless the attribute match specifies a `...'). */ - if (!lambda.formals->ellipsis && attrsUsed != arg.attrs->size()) { - /* Nope, so show the first unexpected argument to the - user. */ - foreach (Bindings::iterator, i, *arg.attrs) - if (lambda.formals->argNames.find(i->name) == lambda.formals->argNames.end()) - throwTypeError("%1% called with unexpected argument `%2%'", lambda, i->name); - abort(); // can't happen - } - } - - nrFunctionCalls++; - if (countCalls) incrFunctionCall(&lambda); - - /* Evaluate the body. This is conditional on showTrace, because - catching exceptions makes this function not tail-recursive. */ - if (settings.showTrace) - try { - lambda.body->eval(*this, env2, v); - } catch (Error & e) { - addErrorPrefix(e, "while evaluating %1%:\n", lambda); - throw; - } - else - fun.lambda.fun->body->eval(*this, env2, v); -} - - -// Lifted out of callFunction() because it creates a temporary that -// prevents tail-call optimisation. -void EvalState::incrFunctionCall(ExprLambda * fun) -{ - functionCalls[fun]++; -} - - -void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) -{ - forceValue(fun); - - if (fun.type != tLambda || !fun.lambda.fun->matchAttrs) { - res = fun; - return; - } - - Value * actualArgs = allocValue(); - mkAttrs(*actualArgs, fun.lambda.fun->formals->formals.size()); - - foreach (Formals::Formals_::iterator, i, fun.lambda.fun->formals->formals) { - Bindings::iterator j = args.find(i->name); - if (j != args.end()) - actualArgs->attrs->push_back(*j); - else if (!i->def) - throwTypeError("cannot auto-call a function that has an argument without a default value (`%1%')", i->name); - } - - actualArgs->attrs->sort(); - - callFunction(fun, *actualArgs, res); -} - - -void ExprWith::eval(EvalState & state, Env & env, Value & v) -{ - Env & env2(state.allocEnv(1)); - env2.up = &env; - env2.prevWith = prevWith; - env2.haveWithAttrs = false; - env2.values[0] = (Value *) attrs; - - body->eval(state, env2, v); -} - - -void ExprIf::eval(EvalState & state, Env & env, Value & v) -{ - (state.evalBool(env, cond) ? then : else_)->eval(state, env, v); -} - - -void ExprAssert::eval(EvalState & state, Env & env, Value & v) -{ - if (!state.evalBool(env, cond)) - throwAssertionError("assertion failed at %1%", pos); - body->eval(state, env, v); -} - - -void ExprOpNot::eval(EvalState & state, Env & env, Value & v) -{ - mkBool(v, !state.evalBool(env, e)); -} - - -void ExprBuiltin::eval(EvalState & state, Env & env, Value & v) -{ - // Not a hot path at all, but would be nice to access state.baseEnv directly - Env *baseEnv = &env; - while (baseEnv->up) baseEnv = baseEnv->up; - Bindings::iterator binding = baseEnv->values[0]->attrs->find(name); - assert(binding != baseEnv->values[0]->attrs->end()); - v = *binding->value; -} - - -void ExprOpEq::eval(EvalState & state, Env & env, Value & v) -{ - Value v1; e1->eval(state, env, v1); - Value v2; e2->eval(state, env, v2); - mkBool(v, state.eqValues(v1, v2)); -} - - -void ExprOpNEq::eval(EvalState & state, Env & env, Value & v) -{ - Value v1; e1->eval(state, env, v1); - Value v2; e2->eval(state, env, v2); - mkBool(v, !state.eqValues(v1, v2)); -} - - -void ExprOpAnd::eval(EvalState & state, Env & env, Value & v) -{ - mkBool(v, state.evalBool(env, e1) && state.evalBool(env, e2)); -} - - -void ExprOpOr::eval(EvalState & state, Env & env, Value & v) -{ - mkBool(v, state.evalBool(env, e1) || state.evalBool(env, e2)); -} - - -void ExprOpImpl::eval(EvalState & state, Env & env, Value & v) -{ - mkBool(v, !state.evalBool(env, e1) || state.evalBool(env, e2)); -} - - -void ExprOpUpdate::eval(EvalState & state, Env & env, Value & v) -{ - Value v1, v2; - state.evalAttrs(env, e1, v1); - state.evalAttrs(env, e2, v2); - - state.nrOpUpdates++; - - if (v1.attrs->size() == 0) { v = v2; return; } - if (v2.attrs->size() == 0) { v = v1; return; } - - state.mkAttrs(v, v1.attrs->size() + v2.attrs->size()); - - /* Merge the sets, preferring values from the second set. Make - sure to keep the resulting vector in sorted order. */ - Bindings::iterator i = v1.attrs->begin(); - Bindings::iterator j = v2.attrs->begin(); - - while (i != v1.attrs->end() && j != v2.attrs->end()) { - if (i->name == j->name) { - v.attrs->push_back(*j); - ++i; ++j; - } - else if (i->name < j->name) - v.attrs->push_back(*i++); - else - v.attrs->push_back(*j++); - } - - while (i != v1.attrs->end()) v.attrs->push_back(*i++); - while (j != v2.attrs->end()) v.attrs->push_back(*j++); - - state.nrOpUpdateValuesCopied += v.attrs->size(); -} - - -void ExprOpConcatLists::eval(EvalState & state, Env & env, Value & v) -{ - Value v1; e1->eval(state, env, v1); - Value v2; e2->eval(state, env, v2); - Value * lists[2] = { &v1, &v2 }; - state.concatLists(v, 2, lists); -} - - -void EvalState::concatLists(Value & v, unsigned int nrLists, Value * * lists) -{ - nrListConcats++; - - Value * nonEmpty = 0; - unsigned int len = 0; - for (unsigned int n = 0; n < nrLists; ++n) { - forceList(*lists[n]); - unsigned int l = lists[n]->list.length; - len += l; - if (l) nonEmpty = lists[n]; - } - - if (nonEmpty && len == nonEmpty->list.length) { - v = *nonEmpty; - return; - } - - mkList(v, len); - for (unsigned int n = 0, pos = 0; n < nrLists; ++n) { - unsigned int l = lists[n]->list.length; - memcpy(v.list.elems + pos, lists[n]->list.elems, l * sizeof(Value *)); - pos += l; - } -} - - -void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) -{ - PathSet context; - std::ostringstream s; - NixInt n = 0; - - bool first = !forceString; - ValueType firstType = tString; - - foreach (vector<Expr *>::iterator, i, *es) { - Value vTmp; - (*i)->eval(state, env, vTmp); - - /* If the first element is a path, then the result will also - be a path, we don't copy anything (yet - that's done later, - since paths are copied when they are used in a derivation), - and none of the strings are allowed to have contexts. */ - if (first) { - firstType = vTmp.type; - first = false; - } - - if (firstType == tInt) { - if (vTmp.type != tInt) - throwEvalError("cannot add %1% to an integer", showType(vTmp)); - n += vTmp.integer; - } else - s << state.coerceToString(vTmp, context, false, firstType == tString); - } - - if (firstType == tInt) - mkInt(v, n); - else if (firstType == tPath) { - if (!context.empty()) - throwEvalError("a string that refers to a store path cannot be appended to a path, in `%1%'", s.str()); - mkPath(v, s.str().c_str()); - } else - mkString(v, s.str(), context); -} - - -void ExprPos::eval(EvalState & state, Env & env, Value & v) -{ - state.mkPos(v, &pos); -} - - -void EvalState::strictForceValue(Value & v) -{ - forceValue(v); - - if (v.type == tAttrs) { - foreach (Bindings::iterator, i, *v.attrs) - strictForceValue(*i->value); - } - - else if (v.type == tList) { - for (unsigned int n = 0; n < v.list.length; ++n) - strictForceValue(*v.list.elems[n]); - } -} - - -NixInt EvalState::forceInt(Value & v) -{ - forceValue(v); - if (v.type != tInt) - throwTypeError("value is %1% while an integer was expected", v); - return v.integer; -} - - -bool EvalState::forceBool(Value & v) -{ - forceValue(v); - if (v.type != tBool) - throwTypeError("value is %1% while a Boolean was expected", v); - return v.boolean; -} - - -void EvalState::forceFunction(Value & v) -{ - forceValue(v); - if (v.type != tLambda && v.type != tPrimOp && v.type != tPrimOpApp) - throwTypeError("value is %1% while a function was expected", v); -} - - -string EvalState::forceString(Value & v) -{ - forceValue(v); - if (v.type != tString) - throwTypeError("value is %1% while a string was expected", v); - return string(v.string.s); -} - - -void copyContext(const Value & v, PathSet & context) -{ - if (v.string.context) - for (const char * * p = v.string.context; *p; ++p) - context.insert(*p); -} - - -string EvalState::forceString(Value & v, PathSet & context) -{ - string s = forceString(v); - copyContext(v, context); - return s; -} - - -string EvalState::forceStringNoCtx(Value & v) -{ - string s = forceString(v); - if (v.string.context) - throwEvalError("the string `%1%' is not allowed to refer to a store path (such as `%2%')", - v.string.s, v.string.context[0]); - return s; -} - - -bool EvalState::isDerivation(Value & v) -{ - if (v.type != tAttrs) return false; - Bindings::iterator i = v.attrs->find(sType); - if (i == v.attrs->end()) return false; - forceValue(*i->value); - if (i->value->type != tString) return false; - return strcmp(i->value->string.s, "derivation") == 0; -} - - -string EvalState::coerceToString(Value & v, PathSet & context, - bool coerceMore, bool copyToStore) -{ - forceValue(v); - - string s; - - if (v.type == tString) { - copyContext(v, context); - return v.string.s; - } - - if (v.type == tPath) { - Path path(canonPath(v.path)); - return copyToStore ? copyPathToStore(context, path) : path; - } - - if (v.type == tAttrs) { - Bindings::iterator i = v.attrs->find(sOutPath); - if (i == v.attrs->end()) throwTypeError("cannot coerce a set to a string"); - return coerceToString(*i->value, context, coerceMore, copyToStore); - } - - if (coerceMore) { - - /* Note that `false' is represented as an empty string for - shell scripting convenience, just like `null'. */ - if (v.type == tBool && v.boolean) return "1"; - if (v.type == tBool && !v.boolean) return ""; - if (v.type == tInt) return int2String(v.integer); - if (v.type == tNull) return ""; - - if (v.type == tList) { - string result; - for (unsigned int n = 0; n < v.list.length; ++n) { - result += coerceToString(*v.list.elems[n], - context, coerceMore, copyToStore); - if (n < v.list.length - 1 - /* !!! not quite correct */ - && (v.list.elems[n]->type != tList || v.list.elems[n]->list.length != 0)) - result += " "; - } - return result; - } - } - - throwTypeError("cannot coerce %1% to a string", v); -} - - -string EvalState::copyPathToStore(PathSet & context, const Path & path) -{ - if (nix::isDerivation(path)) - throwEvalError("file names are not allowed to end in `%1%'", drvExtension); - - Path dstPath; - if (srcToStore[path] != "") - dstPath = srcToStore[path]; - else { - dstPath = settings.readOnlyMode - ? computeStorePathForPath(path).first - : store->addToStore(path, true, htSHA256, defaultPathFilter, repair); - srcToStore[path] = dstPath; - printMsg(lvlChatty, format("copied source `%1%' -> `%2%'") - % path % dstPath); - } - - context.insert(dstPath); - return dstPath; -} - - -Path EvalState::coerceToPath(Value & v, PathSet & context) -{ - string path = coerceToString(v, context, false, false); - if (path == "" || path[0] != '/') - throwEvalError("string `%1%' doesn't represent an absolute path", path); - return path; -} - - -bool EvalState::eqValues(Value & v1, Value & v2) -{ - forceValue(v1); - forceValue(v2); - - /* !!! Hack to support some old broken code that relies on pointer - equality tests between sets. (Specifically, builderDefs calls - uniqList on a list of sets.) Will remove this eventually. */ - if (&v1 == &v2) return true; - - if (v1.type != v2.type) return false; - - switch (v1.type) { - - case tInt: - return v1.integer == v2.integer; - - case tBool: - return v1.boolean == v2.boolean; - - case tString: { - /* Compare both the string and its context. */ - if (strcmp(v1.string.s, v2.string.s) != 0) return false; - const char * * p = v1.string.context, * * q = v2.string.context; - if (!p && !q) return true; - if (!p || !q) return false; - for ( ; *p && *q; ++p, ++q) - if (strcmp(*p, *q) != 0) return false; - if (*p || *q) return false; - return true; - } - - case tPath: - return strcmp(v1.path, v2.path) == 0; - - case tNull: - return true; - - case tList: - if (v1.list.length != v2.list.length) return false; - for (unsigned int n = 0; n < v1.list.length; ++n) - if (!eqValues(*v1.list.elems[n], *v2.list.elems[n])) return false; - return true; - - case tAttrs: { - /* If both sets denote a derivation (type = "derivation"), - then compare their outPaths. */ - if (isDerivation(v1) && isDerivation(v2)) { - Bindings::iterator i = v1.attrs->find(sOutPath); - Bindings::iterator j = v2.attrs->find(sOutPath); - if (i != v1.attrs->end() && j != v2.attrs->end()) - return eqValues(*i->value, *j->value); - } - - if (v1.attrs->size() != v2.attrs->size()) return false; - - /* Otherwise, compare the attributes one by one. */ - Bindings::iterator i, j; - for (i = v1.attrs->begin(), j = v2.attrs->begin(); i != v1.attrs->end(); ++i, ++j) - if (i->name != j->name || !eqValues(*i->value, *j->value)) - return false; - - return true; - } - - /* Functions are incomparable. */ - case tLambda: - case tPrimOp: - case tPrimOpApp: - return false; - - default: - throwEvalError("cannot compare %1% with %2%", showType(v1), showType(v2)); - } -} - - -void EvalState::printStats() -{ - bool showStats = getEnv("NIX_SHOW_STATS", "0") != "0"; - Verbosity v = showStats ? lvlInfo : lvlDebug; - printMsg(v, "evaluation statistics:"); - - struct rusage buf; - getrusage(RUSAGE_SELF, &buf); - float cpuTime = buf.ru_utime.tv_sec + ((float) buf.ru_utime.tv_usec / 1000000); - - printMsg(v, format(" time elapsed: %1%") % cpuTime); - printMsg(v, format(" size of a value: %1%") % sizeof(Value)); - printMsg(v, format(" environments allocated: %1% (%2% bytes)") - % nrEnvs % (nrEnvs * sizeof(Env) + nrValuesInEnvs * sizeof(Value *))); - printMsg(v, format(" list elements: %1% (%2% bytes)") - % nrListElems % (nrListElems * sizeof(Value *))); - printMsg(v, format(" list concatenations: %1%") % nrListConcats); - printMsg(v, format(" values allocated: %1% (%2% bytes)") - % nrValues % (nrValues * sizeof(Value))); - printMsg(v, format(" sets allocated: %1%") % nrAttrsets); - printMsg(v, format(" right-biased unions: %1%") % nrOpUpdates); - printMsg(v, format(" values copied in right-biased unions: %1%") % nrOpUpdateValuesCopied); - printMsg(v, format(" symbols in symbol table: %1%") % symbols.size()); - printMsg(v, format(" size of symbol table: %1%") % symbols.totalSize()); - printMsg(v, format(" number of thunks: %1%") % nrThunks); - printMsg(v, format(" number of thunks avoided: %1%") % nrAvoided); - printMsg(v, format(" number of attr lookups: %1%") % nrLookups); - printMsg(v, format(" number of primop calls: %1%") % nrPrimOpCalls); - printMsg(v, format(" number of function calls: %1%") % nrFunctionCalls); - - if (countCalls) { - v = lvlInfo; - - printMsg(v, format("calls to %1% primops:") % primOpCalls.size()); - typedef std::multimap<unsigned int, Symbol> PrimOpCalls_; - PrimOpCalls_ primOpCalls_; - foreach (PrimOpCalls::iterator, i, primOpCalls) - primOpCalls_.insert(std::pair<unsigned int, Symbol>(i->second, i->first)); - foreach_reverse (PrimOpCalls_::reverse_iterator, i, primOpCalls_) - printMsg(v, format("%1$10d %2%") % i->first % i->second); - - printMsg(v, format("calls to %1% functions:") % functionCalls.size()); - typedef std::multimap<unsigned int, ExprLambda *> FunctionCalls_; - FunctionCalls_ functionCalls_; - foreach (FunctionCalls::iterator, i, functionCalls) - functionCalls_.insert(std::pair<unsigned int, ExprLambda *>(i->second, i->first)); - foreach_reverse (FunctionCalls_::reverse_iterator, i, functionCalls_) - printMsg(v, format("%1$10d %2%") % i->first % i->second->showNamePos()); - - printMsg(v, format("evaluations of %1% attributes:") % attrSelects.size()); - typedef std::multimap<unsigned int, Pos> AttrSelects_; - AttrSelects_ attrSelects_; - foreach (AttrSelects::iterator, i, attrSelects) - attrSelects_.insert(std::pair<unsigned int, Pos>(i->second, i->first)); - foreach_reverse (AttrSelects_::reverse_iterator, i, attrSelects_) - printMsg(v, format("%1$10d %2%") % i->first % i->second); - - } -} - - -} diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh deleted file mode 100644 index 58d1318c2a..0000000000 --- a/src/libexpr/eval.hh +++ /dev/null @@ -1,293 +0,0 @@ -#pragma once - -#include "value.hh" -#include "nixexpr.hh" -#include "symbol-table.hh" -#include "hash.hh" - -#include <map> - -#if HAVE_BOEHMGC -#include <gc/gc_allocator.h> -#endif - - -namespace nix { - - -class EvalState; -struct Attr; - - -/* Sets are represented as a vector of attributes, sorted by symbol - (i.e. pointer to the attribute name in the symbol table). */ -#if HAVE_BOEHMGC -typedef std::vector<Attr, gc_allocator<Attr> > BindingsBase; -#else -typedef std::vector<Attr> BindingsBase; -#endif - - -class Bindings : public BindingsBase -{ -public: - iterator find(const Symbol & name); - void sort(); -}; - - -typedef void (* PrimOpFun) (EvalState & state, Value * * args, Value & v); - - -struct PrimOp -{ - PrimOpFun fun; - unsigned int arity; - Symbol name; - PrimOp(PrimOpFun fun, unsigned int arity, Symbol name) - : fun(fun), arity(arity), name(name) { } -}; - - -struct Env -{ - Env * up; - unsigned short prevWith; // nr of levels up to next `with' environment - bool haveWithAttrs; - Value * values[0]; -}; - - -struct Attr -{ - Symbol name; - Value * value; - Pos * pos; - Attr(Symbol name, Value * value, Pos * pos = &noPos) - : name(name), value(value), pos(pos) { }; - Attr() : pos(&noPos) { }; - bool operator < (const Attr & a) const - { - return name < a.name; - } -}; - - -void mkString(Value & v, const string & s, const PathSet & context = PathSet()); - -void copyContext(const Value & v, PathSet & context); - - -/* Cache for calls to addToStore(); maps source paths to the store - paths. */ -typedef std::map<Path, Path> SrcToStore; - - -std::ostream & operator << (std::ostream & str, const Value & v); - - -class EvalState -{ -public: - SymbolTable symbols; - - const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue, - sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls, - sFile, sLine, sColumn; - Symbol sDerivationNix; - - /* If set, force copying files to the Nix store even if they - already exist there. */ - bool repair; - -private: - SrcToStore srcToStore; - - /* A cache from path names to values. */ -#if HAVE_BOEHMGC - typedef std::map<Path, Value, std::less<Path>, gc_allocator<std::pair<const Path, Value> > > FileEvalCache; -#else - typedef std::map<Path, Value> FileEvalCache; -#endif - FileEvalCache fileEvalCache; - - typedef list<std::pair<string, Path> > SearchPath; - SearchPath searchPath; - SearchPath::iterator searchPathInsertionPoint; - -public: - - EvalState(); - ~EvalState(); - - void addToSearchPath(const string & s, bool warn = false); - - /* Parse a Nix expression from the specified file. */ - Expr * parseExprFromFile(const Path & path); - - /* Parse a Nix expression from the specified string. */ - Expr * parseExprFromString(const string & s, const Path & basePath, StaticEnv & staticEnv); - Expr * parseExprFromString(const string & s, const Path & basePath); - - /* Evaluate an expression read from the given file to normal - form. */ - void evalFile(const Path & path, Value & v); - - void resetFileCache(); - - /* Look up a file in the search path. */ - Path findFile(const string & path); - - /* Evaluate an expression to normal form, storing the result in - value `v'. */ - void eval(Expr * e, Value & v); - - /* Evaluation the expression, then verify that it has the expected - type. */ - inline bool evalBool(Env & env, Expr * e); - inline void evalAttrs(Env & env, Expr * e, Value & v); - - /* If `v' is a thunk, enter it and overwrite `v' with the result - of the evaluation of the thunk. If `v' is a delayed function - application, call the function and overwrite `v' with the - result. Otherwise, this is a no-op. */ - inline void forceValue(Value & v); - - /* Force a value, then recursively force list elements and - attributes. */ - void strictForceValue(Value & v); - - /* Force `v', and then verify that it has the expected type. */ - NixInt forceInt(Value & v); - bool forceBool(Value & v); - inline void forceAttrs(Value & v); - inline void forceList(Value & v); - void forceFunction(Value & v); // either lambda or primop - string forceString(Value & v); - string forceString(Value & v, PathSet & context); - string forceStringNoCtx(Value & v); - - /* Return true iff the value `v' denotes a derivation (i.e. a - set with attribute `type = "derivation"'). */ - bool isDerivation(Value & v); - - /* String coercion. Converts strings, paths and derivations to a - string. If `coerceMore' is set, also converts nulls, integers, - booleans and lists to a string. If `copyToStore' is set, - referenced paths are copied to the Nix store as a side effect. */ - string coerceToString(Value & v, PathSet & context, - bool coerceMore = false, bool copyToStore = true); - - string copyPathToStore(PathSet & context, const Path & path); - - /* Path coercion. Converts strings, paths and derivations to a - path. The result is guaranteed to be a canonicalised, absolute - path. Nothing is copied to the store. */ - Path coerceToPath(Value & v, PathSet & context); - -public: - - /* The base environment, containing the builtin functions and - values. */ - Env & baseEnv; - - /* The same, but used during parsing to resolve variables. */ - StaticEnv staticBaseEnv; // !!! should be private - -private: - - unsigned int baseEnvDispl; - - void createBaseEnv(); - - void addConstant(const string & name, Value & v); - - void addPrimOp(const string & name, - unsigned int arity, PrimOpFun primOp); - -public: - - void getBuiltin(const string & name, Value & v); - -private: - - inline Value * lookupVar(Env * env, const ExprVar & var, bool noEval); - - friend struct ExprVar; - friend struct ExprAttrs; - friend struct ExprLet; - - Expr * parse(const char * text, const Path & path, - const Path & basePath, StaticEnv & staticEnv); - -public: - - /* Do a deep equality test between two values. That is, list - elements and attributes are compared recursively. */ - bool eqValues(Value & v1, Value & v2); - - void callFunction(Value & fun, Value & arg, Value & v); - void callPrimOp(Value & fun, Value & arg, Value & v); - - /* Automatically call a function for which each argument has a - default value or has a binding in the `args' map. */ - void autoCallFunction(Bindings & args, Value & fun, Value & res); - - /* Allocation primitives. */ - Value * allocValue(); - Env & allocEnv(unsigned int size); - - Value * allocAttr(Value & vAttrs, const Symbol & name); - - void mkList(Value & v, unsigned int length); - void mkAttrs(Value & v, unsigned int expected); - void mkThunk_(Value & v, Expr * expr); - void mkPos(Value & v, Pos * pos); - - void concatLists(Value & v, unsigned int nrLists, Value * * lists); - - /* Print statistics. */ - void printStats(); - -private: - - unsigned long nrEnvs; - unsigned long nrValuesInEnvs; - unsigned long nrValues; - unsigned long nrListElems; - unsigned long nrAttrsets; - unsigned long nrOpUpdates; - unsigned long nrOpUpdateValuesCopied; - unsigned long nrListConcats; - unsigned long nrPrimOpCalls; - unsigned long nrFunctionCalls; - - bool countCalls; - - typedef std::map<Symbol, unsigned int> PrimOpCalls; - PrimOpCalls primOpCalls; - - typedef std::map<ExprLambda *, unsigned int> FunctionCalls; - FunctionCalls functionCalls; - - void incrFunctionCall(ExprLambda * fun); - - typedef std::map<Pos, unsigned int> AttrSelects; - AttrSelects attrSelects; - - friend struct ExprOpUpdate; - friend struct ExprOpConcatLists; - friend struct ExprSelect; - friend void prim_getAttr(EvalState & state, Value * * args, Value & v); -}; - - -/* Return a string representing the type of the value `v'. */ -string showType(const Value & v); - - -/* If `path' refers to a directory, then append "/default.nix". */ -Path resolveExprPath(Path path); - - -} diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc deleted file mode 100644 index 0ed644e9bc..0000000000 --- a/src/libexpr/get-drvs.cc +++ /dev/null @@ -1,305 +0,0 @@ -#include "get-drvs.hh" -#include "util.hh" -#include "eval-inline.hh" - -#include <cstring> - - -namespace nix { - - -string DrvInfo::queryDrvPath() -{ - if (drvPath == "" && attrs) { - Bindings::iterator i = attrs->find(state->sDrvPath); - PathSet context; - drvPath = i != attrs->end() ? state->coerceToPath(*i->value, context) : ""; - } - return drvPath; -} - - -string DrvInfo::queryOutPath() -{ - if (outPath == "" && attrs) { - Bindings::iterator i = attrs->find(state->sOutPath); - PathSet context; - outPath = i != attrs->end() ? state->coerceToPath(*i->value, context) : ""; - } - return outPath; -} - - -DrvInfo::Outputs DrvInfo::queryOutputs() -{ - if (outputs.empty()) { - /* Get the âoutputsâ list. */ - Bindings::iterator i; - if (attrs && (i = attrs->find(state->sOutputs)) != attrs->end()) { - state->forceList(*i->value); - - /* For each output... */ - for (unsigned int j = 0; j < i->value->list.length; ++j) { - /* Evaluate the corresponding set. */ - string name = state->forceStringNoCtx(*i->value->list.elems[j]); - Bindings::iterator out = attrs->find(state->symbols.create(name)); - if (out == attrs->end()) continue; // FIXME: throw error? - state->forceAttrs(*out->value); - - /* And evaluate its âoutPathâ attribute. */ - Bindings::iterator outPath = out->value->attrs->find(state->sOutPath); - if (outPath == out->value->attrs->end()) continue; // FIXME: throw error? - PathSet context; - outputs[name] = state->coerceToPath(*outPath->value, context); - } - } else - outputs["out"] = queryOutPath(); - } - return outputs; -} - - -string DrvInfo::queryOutputName() -{ - if (outputName == "" && attrs) { - Bindings::iterator i = attrs->find(state->sOutputName); - outputName = i != attrs->end() ? state->forceStringNoCtx(*i->value) : ""; - } - return outputName; -} - - -Bindings * DrvInfo::getMeta() -{ - if (meta) return meta; - if (!attrs) return 0; - Bindings::iterator a = attrs->find(state->sMeta); - if (a == attrs->end()) return 0; - state->forceAttrs(*a->value); - meta = a->value->attrs; - return meta; -} - - -StringSet DrvInfo::queryMetaNames() -{ - StringSet res; - if (!getMeta()) return res; - foreach (Bindings::iterator, i, *meta) - res.insert(i->name); - return res; -} - - -bool DrvInfo::checkMeta(Value & v) -{ - state->forceValue(v); - if (v.type == tList) { - for (unsigned int n = 0; n < v.list.length; ++n) - if (!checkMeta(*v.list.elems[n])) return false; - return true; - } - else if (v.type == tAttrs) { - Bindings::iterator i = v.attrs->find(state->sOutPath); - if (i != v.attrs->end()) return false; - foreach (Bindings::iterator, i, *v.attrs) - if (!checkMeta(*i->value)) return false; - return true; - } - else return v.type == tInt || v.type == tBool || v.type == tString; -} - - -Value * DrvInfo::queryMeta(const string & name) -{ - if (!getMeta()) return 0; - Bindings::iterator a = meta->find(state->symbols.create(name)); - if (a == meta->end() || !checkMeta(*a->value)) return 0; - return a->value; -} - - -string DrvInfo::queryMetaString(const string & name) -{ - Value * v = queryMeta(name); - if (!v || v->type != tString) return ""; - return v->string.s; -} - - -int DrvInfo::queryMetaInt(const string & name, int def) -{ - Value * v = queryMeta(name); - if (!v) return def; - if (v->type == tInt) return v->integer; - if (v->type == tString) { - /* Backwards compatibility with before we had support for - integer meta fields. */ - int n; - if (string2Int(v->string.s, n)) return n; - } - return def; -} - - -bool DrvInfo::queryMetaBool(const string & name, bool def) -{ - Value * v = queryMeta(name); - if (!v) return def; - if (v->type == tBool) return v->boolean; - if (v->type == tString) { - /* Backwards compatibility with before we had support for - Boolean meta fields. */ - if (strcmp(v->string.s, "true") == 0) return true; - if (strcmp(v->string.s, "false") == 0) return false; - } - return def; -} - - -void DrvInfo::setMeta(const string & name, Value * v) -{ - getMeta(); - Bindings * old = meta; - meta = new Bindings(); - Symbol sym = state->symbols.create(name); - if (old) - foreach (Bindings::iterator, i, *old) - if (i->name != sym) - meta->push_back(*i); - if (v) meta->push_back(Attr(sym, v)); - meta->sort(); -} - - -/* Cache for already considered attrsets. */ -typedef set<Bindings *> Done; - - -/* Evaluate value `v'. If it evaluates to a set of type `derivation', - then put information about it in `drvs' (unless it's already in - `doneExprs'). The result boolean indicates whether it makes sense - for the caller to recursively search for derivations in `v'. */ -static bool getDerivation(EvalState & state, Value & v, - const string & attrPath, DrvInfos & drvs, Done & done, - bool ignoreAssertionFailures) -{ - try { - state.forceValue(v); - if (!state.isDerivation(v)) return true; - - /* Remove spurious duplicates (e.g., a set like `rec { x = - derivation {...}; y = x;}'. */ - if (done.find(v.attrs) != done.end()) return false; - done.insert(v.attrs); - - Bindings::iterator i = v.attrs->find(state.sName); - /* !!! We really would like to have a decent back trace here. */ - if (i == v.attrs->end()) throw TypeError("derivation name missing"); - - Bindings::iterator i2 = v.attrs->find(state.sSystem); - - DrvInfo drv( - state, - state.forceStringNoCtx(*i->value), - attrPath, - i2 == v.attrs->end() ? "unknown" : state.forceStringNoCtx(*i2->value), - v.attrs); - - drvs.push_back(drv); - return false; - - } catch (AssertionError & e) { - if (ignoreAssertionFailures) return false; - throw; - } -} - - -bool getDerivation(EvalState & state, Value & v, DrvInfo & drv, - bool ignoreAssertionFailures) -{ - Done done; - DrvInfos drvs; - getDerivation(state, v, "", drvs, done, ignoreAssertionFailures); - if (drvs.size() != 1) return false; - drv = drvs.front(); - return true; -} - - -static string addToPath(const string & s1, const string & s2) -{ - return s1.empty() ? s2 : s1 + "." + s2; -} - - -static void getDerivations(EvalState & state, Value & vIn, - const string & pathPrefix, Bindings & autoArgs, - DrvInfos & drvs, Done & done, - bool ignoreAssertionFailures) -{ - Value v; - state.autoCallFunction(autoArgs, vIn, v); - - /* Process the expression. */ - if (!getDerivation(state, v, pathPrefix, drvs, done, ignoreAssertionFailures)) ; - - else if (v.type == tAttrs) { - - /* !!! undocumented hackery to support combining channels in - nix-env.cc. */ - bool combineChannels = v.attrs->find(state.symbols.create("_combineChannels")) != v.attrs->end(); - - /* Consider the attributes in sorted order to get more - deterministic behaviour in nix-env operations (e.g. when - there are names clashes between derivations, the derivation - bound to the attribute with the "lower" name should take - precedence). */ - typedef std::map<string, Symbol> SortedSymbols; - SortedSymbols attrs; - foreach (Bindings::iterator, i, *v.attrs) - attrs.insert(std::pair<string, Symbol>(i->name, i->name)); - - foreach (SortedSymbols::iterator, i, attrs) { - startNest(nest, lvlDebug, format("evaluating attribute `%1%'") % i->first); - string pathPrefix2 = addToPath(pathPrefix, i->first); - Value & v2(*v.attrs->find(i->second)->value); - if (combineChannels) - getDerivations(state, v2, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); - else if (getDerivation(state, v2, pathPrefix2, drvs, done, ignoreAssertionFailures)) { - /* If the value of this attribute is itself a set, - should we recurse into it? => Only if it has a - `recurseForDerivations = true' attribute. */ - if (v2.type == tAttrs) { - Bindings::iterator j = v2.attrs->find(state.symbols.create("recurseForDerivations")); - if (j != v2.attrs->end() && state.forceBool(*j->value)) - getDerivations(state, v2, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); - } - } - } - } - - else if (v.type == tList) { - for (unsigned int n = 0; n < v.list.length; ++n) { - startNest(nest, lvlDebug, - format("evaluating list element")); - string pathPrefix2 = addToPath(pathPrefix, (format("%1%") % n).str()); - if (getDerivation(state, *v.list.elems[n], pathPrefix2, drvs, done, ignoreAssertionFailures)) - getDerivations(state, *v.list.elems[n], pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); - } - } - - else throw TypeError("expression does not evaluate to a derivation (or a set or list of those)"); -} - - -void getDerivations(EvalState & state, Value & v, const string & pathPrefix, - Bindings & autoArgs, DrvInfos & drvs, bool ignoreAssertionFailures) -{ - Done done; - getDerivations(state, v, pathPrefix, autoArgs, drvs, done, ignoreAssertionFailures); -} - - -} diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh deleted file mode 100644 index 98f762494a..0000000000 --- a/src/libexpr/get-drvs.hh +++ /dev/null @@ -1,91 +0,0 @@ -#pragma once - -#include "eval.hh" - -#include <string> -#include <map> - - -namespace nix { - - -struct DrvInfo -{ -public: - typedef std::map<string, Path> Outputs; - -private: - EvalState * state; - - string drvPath; - string outPath; - string outputName; - Outputs outputs; - - bool failed; // set if we get an AssertionError - - Bindings * attrs, * meta; - - Bindings * getMeta(); - - bool checkMeta(Value & v); - -public: - string name; - string attrPath; /* path towards the derivation */ - string system; - - DrvInfo(EvalState & state) : state(&state), failed(false), attrs(0), meta(0) { }; - DrvInfo(EvalState & state, const string & name, const string & attrPath, const string & system, Bindings * attrs) - : state(&state), failed(false), attrs(attrs), meta(0), name(name), attrPath(attrPath), system(system) { }; - - string queryDrvPath(); - string queryOutPath(); - string queryOutputName(); - Outputs queryOutputs(); - - StringSet queryMetaNames(); - Value * queryMeta(const string & name); - string queryMetaString(const string & name); - int queryMetaInt(const string & name, int def); - bool queryMetaBool(const string & name, bool def); - void setMeta(const string & name, Value * v); - - /* - MetaInfo queryMetaInfo(EvalState & state) const; - MetaValue queryMetaInfo(EvalState & state, const string & name) const; - */ - - void setDrvPath(const string & s) - { - drvPath = s; - } - - void setOutPath(const string & s) - { - outPath = s; - } - - void setFailed() { failed = true; }; - bool hasFailed() { return failed; }; -}; - - -#if HAVE_BOEHMGC -typedef list<DrvInfo, traceable_allocator<DrvInfo> > DrvInfos; -#else -typedef list<DrvInfo> DrvInfos; -#endif - - -/* If value `v' denotes a derivation, store information about the - derivation in `drv' and return true. Otherwise, return false. */ -bool getDerivation(EvalState & state, Value & v, DrvInfo & drv, - bool ignoreAssertionFailures); - -void getDerivations(EvalState & state, Value & v, const string & pathPrefix, - Bindings & autoArgs, DrvInfos & drvs, - bool ignoreAssertionFailures); - - -} diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l deleted file mode 100644 index 911850cc5b..0000000000 --- a/src/libexpr/lexer.l +++ /dev/null @@ -1,193 +0,0 @@ -%option reentrant bison-bridge bison-locations -%option noyywrap -%option never-interactive - - -%x STRING -%x IND_STRING - - -%{ -#include "nixexpr.hh" -#include "parser-tab.hh" - -using namespace nix; - -namespace nix { - - -static void initLoc(YYLTYPE * loc) -{ - loc->first_line = loc->last_line = 1; - loc->first_column = loc->last_column = 1; -} - - -static void adjustLoc(YYLTYPE * loc, const char * s, size_t len) -{ - loc->first_line = loc->last_line; - loc->first_column = loc->last_column; - - while (len--) { - switch (*s++) { - case '\r': - if (*s == '\n') /* cr/lf */ - s++; - /* fall through */ - case '\n': - ++loc->last_line; - loc->last_column = 1; - break; - default: - ++loc->last_column; - } - } -} - - -static Expr * unescapeStr(SymbolTable & symbols, const char * s) -{ - string t; - char c; - while ((c = *s++)) { - if (c == '\\') { - assert(*s); - c = *s++; - if (c == 'n') t += '\n'; - else if (c == 'r') t += '\r'; - else if (c == 't') t += '\t'; - else t += c; - } - else if (c == '\r') { - /* Normalise CR and CR/LF into LF. */ - t += '\n'; - if (*s == '\n') s++; /* cr/lf */ - } - else t += c; - } - return new ExprString(symbols.create(t)); -} - - -} - -#define YY_USER_INIT initLoc(yylloc) -#define YY_USER_ACTION adjustLoc(yylloc, yytext, yyleng); - -%} - - -ID [a-zA-Z\_][a-zA-Z0-9\_\'\-]* -INT [0-9]+ -PATH [a-zA-Z0-9\.\_\-\+]*(\/[a-zA-Z0-9\.\_\-\+]+)+ -SPATH \<[a-zA-Z0-9\.\_\-\+]+(\/[a-zA-Z0-9\.\_\-\+]+)*\> -URI [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~\*\']+ - - -%% - - -if { return IF; } -then { return THEN; } -else { return ELSE; } -assert { return ASSERT; } -with { return WITH; } -let { return LET; } -in { return IN; } -rec { return REC; } -inherit { return INHERIT; } -or { return OR_KW; } -\.\.\. { return ELLIPSIS; } - -\=\= { return EQ; } -\!\= { return NEQ; } -\<\= { return LEQ; } -\>\= { return GEQ; } -\&\& { return AND; } -\|\| { return OR; } -\-\> { return IMPL; } -\/\/ { return UPDATE; } -\+\+ { return CONCAT; } - -{ID} { yylval->id = strdup(yytext); return ID; } -{INT} { errno = 0; - yylval->n = strtol(yytext, 0, 10); - if (errno != 0) - throw ParseError(format("invalid integer `%1%'") % yytext); - return INT; - } - -\$\{ { return DOLLAR_CURLY; } - -\" { BEGIN(STRING); return '"'; } -<STRING>([^\$\"\\]|\$[^\{\"]|\\.)+ { - /* !!! Not quite right: we want a follow restriction on - "$", it shouldn't be followed by a "{". Right now - "$\"" will be consumed as part of a string, rather - than a "$" followed by the string terminator. - Disallow "$\"" for now. */ - yylval->e = unescapeStr(data->symbols, yytext); - return STR; - } -<STRING>\$\{ { BEGIN(INITIAL); return DOLLAR_CURLY; } -<STRING>\" { BEGIN(INITIAL); return '"'; } -<STRING>. return yytext[0]; /* just in case: shouldn't be reached */ - -\'\'(\ *\n)? { BEGIN(IND_STRING); return IND_STRING_OPEN; } -<IND_STRING>([^\$\']|\$[^\{\']|\'[^\'\$])+ { - yylval->e = new ExprIndStr(yytext); - return IND_STR; - } -<IND_STRING>\'\'\$ { - yylval->e = new ExprIndStr("$"); - return IND_STR; - } -<IND_STRING>\'\'\' { - yylval->e = new ExprIndStr("''"); - return IND_STR; - } -<IND_STRING>\'\'\\. { - yylval->e = unescapeStr(data->symbols, yytext + 2); - return IND_STR; - } -<IND_STRING>\$\{ { BEGIN(INITIAL); return DOLLAR_CURLY; } -<IND_STRING>\'\' { BEGIN(INITIAL); return IND_STRING_CLOSE; } -<IND_STRING>\' { - yylval->e = new ExprIndStr("'"); - return IND_STR; - } -<IND_STRING>. return yytext[0]; /* just in case: shouldn't be reached */ - -{PATH} { yylval->path = strdup(yytext); return PATH; } -{SPATH} { yylval->path = strdup(yytext); return SPATH; } -{URI} { yylval->uri = strdup(yytext); return URI; } - -[ \t\r\n]+ /* eat up whitespace */ -\#[^\r\n]* /* single-line comments */ -\/\*([^*]|\*[^\/])*\*\/ /* long comments */ - -. return yytext[0]; - - -%% - - -namespace nix { - -/* Horrible, disgusting hack: allow the parser to set the scanner - start condition back to STRING. Necessary in interpolations like - "foo${expr}bar"; after the close brace we have to go back to the - STRING state. */ -void backToString(yyscan_t scanner) -{ - struct yyguts_t * yyg = (struct yyguts_t *) scanner; - BEGIN(STRING); -} - -void backToIndString(yyscan_t scanner) -{ - struct yyguts_t * yyg = (struct yyguts_t *) scanner; - BEGIN(IND_STRING); -} - -} diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk deleted file mode 100644 index b3b4086916..0000000000 --- a/src/libexpr/local.mk +++ /dev/null @@ -1,28 +0,0 @@ -libraries += libexpr - -libexpr_NAME = libnixexpr - -libexpr_DIR := $(d) - -libexpr_SOURCES := $(wildcard $(d)/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc - -libexpr_LIBS = libutil libstore libformat - -# The dependency on libgc must be propagated (i.e. meaning that -# programs/libraries that use libexpr must explicitly pass -lgc), -# because inline functions in libexpr's header files call libgc. -libexpr_LDFLAGS_PROPAGATED = $(BDW_GC_LIBS) - -$(d)/parser-tab.cc $(d)/parser-tab.hh: $(d)/parser.y - $(trace-gen) bison -v -o $(libexpr_DIR)/parser-tab.cc $< -d - -$(d)/lexer-tab.cc $(d)/lexer-tab.hh: $(d)/lexer.l - $(trace-gen) flex --outfile $(libexpr_DIR)/lexer-tab.cc --header-file=$(libexpr_DIR)/lexer-tab.hh $< - -$(d)/lexer-tab.o: $(d)/lexer-tab.hh $(d)/parser-tab.hh - -$(d)/parser-tab.o: $(d)/lexer-tab.hh $(d)/parser-tab.hh - -clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh - -dist-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh diff --git a/src/libexpr/names.cc b/src/libexpr/names.cc deleted file mode 100644 index 781c2b6468..0000000000 --- a/src/libexpr/names.cc +++ /dev/null @@ -1,104 +0,0 @@ -#include "names.hh" -#include "util.hh" - - -namespace nix { - - -DrvName::DrvName() -{ - name = ""; -} - - -/* Parse a derivation name. The `name' part of a derivation name is - everything up to but not including the first dash *not* followed by - a letter. The `version' part is the rest (excluding the separating - dash). E.g., `apache-httpd-2.0.48' is parsed to (`apache-httpd', - '2.0.48'). */ -DrvName::DrvName(const string & s) : hits(0) -{ - name = fullName = s; - for (unsigned int i = 0; i < s.size(); ++i) { - /* !!! isalpha/isdigit are affected by the locale. */ - if (s[i] == '-' && i + 1 < s.size() && !isalpha(s[i + 1])) { - name = string(s, 0, i); - version = string(s, i + 1); - break; - } - } -} - - -bool DrvName::matches(DrvName & n) -{ - if (name != "*" && name != n.name) return false; - if (version != "" && version != n.version) return false; - return true; -} - - -static string nextComponent(string::const_iterator & p, - const string::const_iterator end) -{ - /* Skip any dots and dashes (component separators). */ - while (p != end && (*p == '.' || *p == '-')) ++p; - - if (p == end) return ""; - - /* If the first character is a digit, consume the longest sequence - of digits. Otherwise, consume the longest sequence of - non-digit, non-separator characters. */ - string s; - if (isdigit(*p)) - while (p != end && isdigit(*p)) s += *p++; - else - while (p != end && (!isdigit(*p) && *p != '.' && *p != '-')) - s += *p++; - - return s; -} - - -static bool componentsLT(const string & c1, const string & c2) -{ - int n1, n2; - bool c1Num = string2Int(c1, n1), c2Num = string2Int(c2, n2); - - if (c1Num && c2Num) return n1 < n2; - else if (c1 == "" && c2Num) return true; - else if (c1 == "pre" && c2 != "pre") return true; - else if (c2 == "pre") return false; - /* Assume that `2.3a' < `2.3.1'. */ - else if (c2Num) return true; - else if (c1Num) return false; - else return c1 < c2; -} - - -int compareVersions(const string & v1, const string & v2) -{ - string::const_iterator p1 = v1.begin(); - string::const_iterator p2 = v2.begin(); - - while (p1 != v1.end() || p2 != v2.end()) { - string c1 = nextComponent(p1, v1.end()); - string c2 = nextComponent(p2, v2.end()); - if (componentsLT(c1, c2)) return -1; - else if (componentsLT(c2, c1)) return 1; - } - - return 0; -} - - -DrvNames drvNamesFromArgs(const Strings & opArgs) -{ - DrvNames result; - foreach (Strings::const_iterator, i, opArgs) - result.push_back(DrvName(*i)); - return result; -} - - -} diff --git a/src/libexpr/names.hh b/src/libexpr/names.hh deleted file mode 100644 index ebe113e82a..0000000000 --- a/src/libexpr/names.hh +++ /dev/null @@ -1,24 +0,0 @@ -#pragma once - -#include "types.hh" - -namespace nix { - -struct DrvName -{ - string fullName; - string name; - string version; - unsigned int hits; - - DrvName(); - DrvName(const string & s); - bool matches(DrvName & n); -}; - -typedef list<DrvName> DrvNames; - -int compareVersions(const string & v1, const string & v2); -DrvNames drvNamesFromArgs(const Strings & opArgs); - -} diff --git a/src/libexpr/nix.sdf b/src/libexpr/nix.sdf deleted file mode 100644 index 42fb21c3ba..0000000000 --- a/src/libexpr/nix.sdf +++ /dev/null @@ -1,141 +0,0 @@ -%% Note: this SDF grammar is no longer used in the Nix expression -%% parser and may not be up to date. - -definition - - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% Top level syntax. - -module Main -imports Nix-Exprs Nix-Layout - - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% Expressions. - -module Nix-Exprs -imports Nix-Lexicals -exports - sorts Expr Formal Bind ExprList - context-free start-symbols Expr - context-free syntax - - Id -> Expr {cons("Var")} - Int -> Expr {cons("Int")} - Str -> Expr {cons("Str")} - Uri -> Expr {cons("Uri")} - Path -> Expr {cons("Path")} - - "(" Expr ")" -> Expr {bracket} - - Expr Expr -> Expr {cons("Call"), left} - - Id ":" Expr -> Expr {cons("Function1")} - "{" {Formal ","}* "}" ":" Expr -> Expr {cons("Function")} - Id -> Formal {cons("NoDefFormal")} - Id "?" Expr -> Formal {cons("DefFormal")} - - "assert" Expr ";" Expr -> Expr {cons("Assert")} - - "with" Expr ";" Expr -> Expr {cons("With")} - - "rec" "{" Bind* "}" -> Expr {cons("Rec")} - "let" Bind* "in" Expr -> Expr {cons("Let")} - "let" "{" Bind* "}" -> Expr {cons("LetRec")} - "{" Bind* "}" -> Expr {cons("Attrs")} - - Id "=" Expr ";" -> Bind {cons("Bind")} - "inherit" ("(" Expr ")")? Id* ";" -> Bind {cons("Inherit")} - - "[" ExprList "]" -> Expr {cons("List")} - -> ExprList {cons("ExprNil")} - Expr ExprList -> ExprList {cons("ExprCons")} - - Expr "." Id -> Expr {cons("Select")} - - "if" Expr "then" Expr "else" Expr -> Expr {cons("If")} - - Expr "==" Expr -> Expr {cons("OpEq"), non-assoc} - Expr "!=" Expr -> Expr {cons("OpNEq"), non-assoc} - - "!" Expr -> Expr {cons("OpNot")} - Expr "&&" Expr -> Expr {cons("OpAnd"), right} - Expr "||" Expr -> Expr {cons("OpOr"), right} - Expr "->" Expr -> Expr {cons("OpImpl"), right} - - Expr "//" Expr -> Expr {cons("OpUpdate"), right} - Expr "~" Expr -> Expr {cons("SubPath"), non-assoc} - Expr "?" Id -> Expr {cons("OpHasAttr")} - Expr "+" Expr -> Expr {cons("OpPlus"), left} - Expr "++" Expr -> Expr {cons("OpConcat"), right} - - context-free priorities - - Expr "." Id -> Expr - > Expr ExprList -> ExprList - > Expr Expr -> Expr - > Expr "~" Expr -> Expr - > Expr "?" Id -> Expr - > Expr "++" Expr -> Expr - > Expr "+" Expr -> Expr - > "!" Expr -> Expr - > Expr "//" Expr -> Expr - > { Expr "==" Expr -> Expr - Expr "!=" Expr -> Expr - } - > Expr "&&" Expr -> Expr - > Expr "||" Expr -> Expr - > Expr "->" Expr -> Expr - > "if" Expr "then" Expr "else" Expr -> Expr - > "assert" Expr ";" Expr -> Expr - > "with" Expr ";" Expr -> Expr - > Id ":" Expr -> Expr - > "{" {Formal ","}* "}" ":" Expr -> Expr - - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% Lexical syntax. - -module Nix-Lexicals -exports - sorts Id Int Str Path Uri - lexical syntax - [a-zA-Z\_][a-zA-Z0-9\_\']* -> Id - "rec" | "let" | "if" | "then" | "else" | "assert" | "with" | "inherit" -> Id {reject} - - [0-9]+ -> Int - - "\"" (~[\"\\] | ("\\" ~[]) )* "\"" -> Str - "''" (~[\"\\] | ("\\" ~[]) )* "''" -> Str - - [a-zA-Z0-9\.\_\-\+]* ("/"[a-zA-Z0-9\.\_\-\+]+)+ -> Path - - [a-zA-Z] [a-zA-Z0-9\+\-\.]* ":" [a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~\*\']+ -> Uri - - lexical restrictions - Id -/- [a-zA-Z0-9\_\'] - Int -/- [0-9] - Path -/- [a-zA-Z0-9\.\_\-\+\/] - Uri -/- [a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~\*\'] - "rec" "let" "if" "then" "else" "assert" "with" "inherit" -/- [A-Za-z0-9\_\'] - - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% Layout. - -module Nix-Layout -exports - sorts HashComment Asterisk Comment - lexical syntax - [\ \t\n] -> LAYOUT - HashComment -> LAYOUT - Comment -> LAYOUT - "#" ~[\n]* -> HashComment - "/*" ( ~[\*] | Asterisk )* "*/" -> Comment - [\*] ~[\/] -> Asterisk - lexical restrictions - HashComment -/- ~[\n] - context-free restrictions - LAYOUT? -/- [\ \t\n\#] - LAYOUT? -/- [\/].[\*] diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc deleted file mode 100644 index bca2b7913b..0000000000 --- a/src/libexpr/nixexpr.cc +++ /dev/null @@ -1,392 +0,0 @@ -#include "nixexpr.hh" -#include "derivations.hh" -#include "util.hh" - -#include <cstdlib> - - -namespace nix { - - -/* Displaying abstract syntax trees. */ - -std::ostream & operator << (std::ostream & str, Expr & e) -{ - e.show(str); - return str; -} - -void Expr::show(std::ostream & str) -{ - abort(); -} - -void ExprInt::show(std::ostream & str) -{ - str << n; -} - -void ExprString::show(std::ostream & str) -{ - str << "\"" << s << "\""; // !!! escaping -} - -void ExprPath::show(std::ostream & str) -{ - str << s; -} - -void ExprVar::show(std::ostream & str) -{ - str << name; -} - -void ExprSelect::show(std::ostream & str) -{ - str << "(" << *e << ")." << showAttrPath(attrPath); - if (def) str << " or " << *def; -} - -void ExprOpHasAttr::show(std::ostream & str) -{ - str << "(" << *e << ") ? " << showAttrPath(attrPath); -} - -void ExprAttrs::show(std::ostream & str) -{ - if (recursive) str << "rec "; - str << "{ "; - foreach (AttrDefs::iterator, i, attrs) - if (i->second.inherited) - str << "inherit " << i->first << " " << "; "; - else - str << i->first << " = " << *i->second.e << "; "; - foreach (DynamicAttrDefs::iterator, i, dynamicAttrs) - str << "\"${" << *i->nameExpr << "}\" = " << *i->valueExpr << "; "; - str << "}"; -} - -void ExprList::show(std::ostream & str) -{ - str << "[ "; - foreach (vector<Expr *>::iterator, i, elems) - str << "(" << **i << ") "; - str << "]"; -} - -void ExprLambda::show(std::ostream & str) -{ - str << "("; - if (matchAttrs) { - str << "{ "; - bool first = true; - foreach (Formals::Formals_::iterator, i, formals->formals) { - if (first) first = false; else str << ", "; - str << i->name; - if (i->def) str << " ? " << *i->def; - } - str << " }"; - if (!arg.empty()) str << " @ "; - } - if (!arg.empty()) str << arg; - str << ": " << *body << ")"; -} - -void ExprLet::show(std::ostream & str) -{ - str << "let "; - foreach (ExprAttrs::AttrDefs::iterator, i, attrs->attrs) - if (i->second.inherited) - str << "inherit " << i->first << "; "; - else - str << i->first << " = " << *i->second.e << "; "; - str << "in " << *body; -} - -void ExprWith::show(std::ostream & str) -{ - str << "with " << *attrs << "; " << *body; -} - -void ExprIf::show(std::ostream & str) -{ - str << "if " << *cond << " then " << *then << " else " << *else_; -} - -void ExprAssert::show(std::ostream & str) -{ - str << "assert " << *cond << "; " << *body; -} - -void ExprOpNot::show(std::ostream & str) -{ - str << "! " << *e; -} - -void ExprBuiltin::show(std::ostream & str) -{ - str << "builtins." << name; -} - -void ExprConcatStrings::show(std::ostream & str) -{ - bool first = true; - foreach (vector<Expr *>::iterator, i, *es) { - if (first) first = false; else str << " + "; - str << **i; - } -} - -void ExprPos::show(std::ostream & str) -{ - str << "__curPos"; -} - - -std::ostream & operator << (std::ostream & str, const Pos & pos) -{ - if (!pos.line) - str << "undefined position"; - else - str << (format("`%1%:%2%:%3%'") % pos.file % pos.line % pos.column).str(); - return str; -} - - -string showAttrPath(const AttrPath & attrPath) -{ - std::ostringstream out; - bool first = true; - foreach (AttrPath::const_iterator, i, attrPath) { - if (!first) - out << '.'; - else - first = false; - if (i->symbol.set()) - out << i->symbol; - else - out << "\"${" << *i->expr << "}\""; - } - return out.str(); -} - - -Pos noPos; - - -/* Computing levels/displacements for variables. */ - -void Expr::bindVars(const StaticEnv & env) -{ - abort(); -} - -void ExprInt::bindVars(const StaticEnv & env) -{ -} - -void ExprString::bindVars(const StaticEnv & env) -{ -} - -void ExprPath::bindVars(const StaticEnv & env) -{ -} - -void ExprVar::bindVars(const StaticEnv & env) -{ - /* Check whether the variable appears in the environment. If so, - set its level and displacement. */ - const StaticEnv * curEnv; - unsigned int level; - int withLevel = -1; - for (curEnv = &env, level = 0; curEnv; curEnv = curEnv->up, level++) { - if (curEnv->isWith) { - if (withLevel == -1) withLevel = level; - } else { - StaticEnv::Vars::const_iterator i = curEnv->vars.find(name); - if (i != curEnv->vars.end()) { - fromWith = false; - this->level = level; - displ = i->second; - return; - } - } - } - - /* Otherwise, the variable must be obtained from the nearest - enclosing `with'. If there is no `with', then we can issue an - "undefined variable" error now. */ - if (withLevel == -1) throw UndefinedVarError(format("undefined variable `%1%' at %2%") % name % pos); - - fromWith = true; - this->level = withLevel; -} - -void ExprSelect::bindVars(const StaticEnv & env) -{ - e->bindVars(env); - if (def) def->bindVars(env); - foreach (AttrPath::iterator, i, attrPath) - if (!i->symbol.set()) - i->expr->bindVars(env); -} - -void ExprOpHasAttr::bindVars(const StaticEnv & env) -{ - e->bindVars(env); - foreach (AttrPath::iterator, i, attrPath) - if (!i->symbol.set()) - i->expr->bindVars(env); -} - -void ExprAttrs::bindVars(const StaticEnv & env) -{ - const StaticEnv * dynamicEnv = &env; - StaticEnv newEnv(false, &env); - - if (recursive) { - dynamicEnv = &newEnv; - - unsigned int displ = 0; - foreach (AttrDefs::iterator, i, attrs) - newEnv.vars[i->first] = i->second.displ = displ++; - - foreach (AttrDefs::iterator, i, attrs) - i->second.e->bindVars(i->second.inherited ? env : newEnv); - } - - else - foreach (AttrDefs::iterator, i, attrs) - i->second.e->bindVars(env); - - foreach (DynamicAttrDefs::iterator, i, dynamicAttrs) { - i->nameExpr->bindVars(*dynamicEnv); - i->valueExpr->bindVars(*dynamicEnv); - } -} - -void ExprList::bindVars(const StaticEnv & env) -{ - foreach (vector<Expr *>::iterator, i, elems) - (*i)->bindVars(env); -} - -void ExprLambda::bindVars(const StaticEnv & env) -{ - StaticEnv newEnv(false, &env); - - unsigned int displ = 0; - - if (!arg.empty()) newEnv.vars[arg] = displ++; - - if (matchAttrs) { - foreach (Formals::Formals_::iterator, i, formals->formals) - newEnv.vars[i->name] = displ++; - - foreach (Formals::Formals_::iterator, i, formals->formals) - if (i->def) i->def->bindVars(newEnv); - } - - body->bindVars(newEnv); -} - -void ExprLet::bindVars(const StaticEnv & env) -{ - StaticEnv newEnv(false, &env); - - unsigned int displ = 0; - foreach (ExprAttrs::AttrDefs::iterator, i, attrs->attrs) - newEnv.vars[i->first] = i->second.displ = displ++; - - foreach (ExprAttrs::AttrDefs::iterator, i, attrs->attrs) - i->second.e->bindVars(i->second.inherited ? env : newEnv); - - body->bindVars(newEnv); -} - -void ExprWith::bindVars(const StaticEnv & env) -{ - /* Does this `with' have an enclosing `with'? If so, record its - level so that `lookupVar' can look up variables in the previous - `with' if this one doesn't contain the desired attribute. */ - const StaticEnv * curEnv; - unsigned int level; - prevWith = 0; - for (curEnv = &env, level = 1; curEnv; curEnv = curEnv->up, level++) - if (curEnv->isWith) { - prevWith = level; - break; - } - - attrs->bindVars(env); - StaticEnv newEnv(true, &env); - body->bindVars(newEnv); -} - -void ExprIf::bindVars(const StaticEnv & env) -{ - cond->bindVars(env); - then->bindVars(env); - else_->bindVars(env); -} - -void ExprAssert::bindVars(const StaticEnv & env) -{ - cond->bindVars(env); - body->bindVars(env); -} - -void ExprOpNot::bindVars(const StaticEnv & env) -{ - e->bindVars(env); -} - -void ExprBuiltin::bindVars(const StaticEnv & env) -{ -} - -void ExprConcatStrings::bindVars(const StaticEnv & env) -{ - foreach (vector<Expr *>::iterator, i, *es) - (*i)->bindVars(env); -} - -void ExprPos::bindVars(const StaticEnv & env) -{ -} - - -/* Storing function names. */ - -void Expr::setName(Symbol & name) -{ -} - - -void ExprLambda::setName(Symbol & name) -{ - this->name = name; - body->setName(name); -} - - -string ExprLambda::showNamePos() const -{ - return (format("%1% at %2%") % (name.set() ? "`" + (string) name + "'" : "an anonymous function") % pos).str(); -} - - - -/* Symbol table. */ - -size_t SymbolTable::totalSize() const -{ - size_t n = 0; - foreach (Symbols::const_iterator, i, symbols) - n += i->size(); - return n; -} - - -} diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh deleted file mode 100644 index a5c5d0533c..0000000000 --- a/src/libexpr/nixexpr.hh +++ /dev/null @@ -1,332 +0,0 @@ -#pragma once - -#include "value.hh" -#include "symbol-table.hh" - -#include <map> - - -namespace nix { - - -MakeError(EvalError, Error) -MakeError(ParseError, Error) -MakeError(AssertionError, EvalError) -MakeError(ThrownError, AssertionError) -MakeError(Abort, EvalError) -MakeError(TypeError, EvalError) -MakeError(ImportError, EvalError) // error building an imported derivation -MakeError(UndefinedVarError, Error) - - -/* Position objects. */ - -struct Pos -{ - Symbol file; - unsigned int line, column; - Pos() : line(0), column(0) { }; - Pos(const Symbol & file, unsigned int line, unsigned int column) - : file(file), line(line), column(column) { }; - bool operator < (const Pos & p2) const - { - if (!line) return p2.line; - if (!p2.line) return false; - int d = ((string) file).compare((string) p2.file); - if (d < 0) return true; - if (d > 0) return false; - if (line < p2.line) return true; - if (line > p2.line) return false; - return column < p2.column; - } -}; - -extern Pos noPos; - -std::ostream & operator << (std::ostream & str, const Pos & pos); - - -struct Env; -struct Value; -class EvalState; -struct StaticEnv; -struct ExprConcatStrings; - - -/* An attribute path is a sequence of attribute names. */ -struct AttrName -{ - Symbol symbol; - ExprConcatStrings * expr; - AttrName(const Symbol & s) : symbol(s) {}; - AttrName(ExprConcatStrings * e) : expr(e) {}; -}; - -typedef std::vector<AttrName> AttrPath; - -string showAttrPath(const AttrPath & attrPath); - - -/* Abstract syntax of Nix expressions. */ - -struct Expr -{ - virtual ~Expr() { }; - virtual void show(std::ostream & str); - virtual void bindVars(const StaticEnv & env); - virtual void eval(EvalState & state, Env & env, Value & v); - virtual Value * maybeThunk(EvalState & state, Env & env); - virtual void setName(Symbol & name); -}; - -std::ostream & operator << (std::ostream & str, Expr & e); - -#define COMMON_METHODS \ - void show(std::ostream & str); \ - void eval(EvalState & state, Env & env, Value & v); \ - void bindVars(const StaticEnv & env); - -struct ExprInt : Expr -{ - NixInt n; - Value v; - ExprInt(NixInt n) : n(n) { mkInt(v, n); }; - COMMON_METHODS - Value * maybeThunk(EvalState & state, Env & env); -}; - -struct ExprString : Expr -{ - Symbol s; - Value v; - ExprString(const Symbol & s) : s(s) { mkString(v, s); }; - COMMON_METHODS - Value * maybeThunk(EvalState & state, Env & env); -}; - -/* Temporary class used during parsing of indented strings. */ -struct ExprIndStr : Expr -{ - string s; - ExprIndStr(const string & s) : s(s) { }; -}; - -struct ExprPath : Expr -{ - string s; - Value v; - ExprPath(const string & s) : s(s) { mkPathNoCopy(v, this->s.c_str()); }; - COMMON_METHODS - Value * maybeThunk(EvalState & state, Env & env); -}; - -struct ExprVar : Expr -{ - Pos pos; - Symbol name; - - /* Whether the variable comes from an environment (e.g. a rec, let - or function argument) or from a "with". */ - bool fromWith; - - /* In the former case, the value is obtained by going `level' - levels up from the current environment and getting the - `displ'th value in that environment. In the latter case, the - value is obtained by getting the attribute named `name' from - the set stored in the environment that is `level' levels up - from the current one.*/ - unsigned int level; - unsigned int displ; - - ExprVar(const Pos & pos, const Symbol & name) : pos(pos), name(name) { }; - COMMON_METHODS - Value * maybeThunk(EvalState & state, Env & env); -}; - -struct ExprSelect : Expr -{ - Expr * e, * def; - AttrPath attrPath; - ExprSelect(Expr * e, const AttrPath & attrPath, Expr * def) : e(e), def(def), attrPath(attrPath) { }; - ExprSelect(Expr * e, const Symbol & name) : e(e), def(0) { attrPath.push_back(AttrName(name)); }; - COMMON_METHODS -}; - -struct ExprOpHasAttr : Expr -{ - Expr * e; - AttrPath attrPath; - ExprOpHasAttr(Expr * e, const AttrPath & attrPath) : e(e), attrPath(attrPath) { }; - COMMON_METHODS -}; - -struct ExprAttrs : Expr -{ - bool recursive; - struct AttrDef { - bool inherited; - Expr * e; - Pos pos; - unsigned int displ; // displacement - AttrDef(Expr * e, const Pos & pos, bool inherited=false) : inherited(inherited), e(e), pos(pos) { }; - AttrDef() { }; - }; - typedef std::map<Symbol, AttrDef> AttrDefs; - AttrDefs attrs; - struct DynamicAttrDef { - ExprConcatStrings * nameExpr; - Expr * valueExpr; - Pos pos; - DynamicAttrDef(ExprConcatStrings * nameExpr, Expr * valueExpr, const Pos & pos) : nameExpr(nameExpr), valueExpr(valueExpr), pos(pos) { }; - }; - typedef std::vector<DynamicAttrDef> DynamicAttrDefs; - DynamicAttrDefs dynamicAttrs; - ExprAttrs() : recursive(false) { }; - COMMON_METHODS -}; - -struct ExprList : Expr -{ - std::vector<Expr *> elems; - ExprList() { }; - COMMON_METHODS -}; - -struct Formal -{ - Symbol name; - Expr * def; - Formal(const Symbol & name, Expr * def) : name(name), def(def) { }; -}; - -struct Formals -{ - typedef std::list<Formal> Formals_; - Formals_ formals; - std::set<Symbol> argNames; // used during parsing - bool ellipsis; -}; - -struct ExprLambda : Expr -{ - Pos pos; - Symbol name; - Symbol arg; - bool matchAttrs; - Formals * formals; - Expr * body; - ExprLambda(const Pos & pos, const Symbol & arg, bool matchAttrs, Formals * formals, Expr * body) - : pos(pos), arg(arg), matchAttrs(matchAttrs), formals(formals), body(body) - { - if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end()) - throw ParseError(format("duplicate formal function argument `%1%' at %2%") - % arg % pos); - }; - void setName(Symbol & name); - string showNamePos() const; - COMMON_METHODS -}; - -struct ExprLet : Expr -{ - ExprAttrs * attrs; - Expr * body; - ExprLet(ExprAttrs * attrs, Expr * body) : attrs(attrs), body(body) { }; - COMMON_METHODS -}; - -struct ExprWith : Expr -{ - Pos pos; - Expr * attrs, * body; - unsigned int prevWith; - ExprWith(const Pos & pos, Expr * attrs, Expr * body) : pos(pos), attrs(attrs), body(body) { }; - COMMON_METHODS -}; - -struct ExprIf : Expr -{ - Expr * cond, * then, * else_; - ExprIf(Expr * cond, Expr * then, Expr * else_) : cond(cond), then(then), else_(else_) { }; - COMMON_METHODS -}; - -struct ExprAssert : Expr -{ - Pos pos; - Expr * cond, * body; - ExprAssert(const Pos & pos, Expr * cond, Expr * body) : pos(pos), cond(cond), body(body) { }; - COMMON_METHODS -}; - -struct ExprOpNot : Expr -{ - Expr * e; - ExprOpNot(Expr * e) : e(e) { }; - COMMON_METHODS -}; - -struct ExprBuiltin : Expr -{ - Symbol name; - ExprBuiltin(const Symbol & name) : name(name) { }; - COMMON_METHODS -}; - -#define MakeBinOp(name, s) \ - struct Expr##name : Expr \ - { \ - Expr * e1, * e2; \ - Expr##name(Expr * e1, Expr * e2) : e1(e1), e2(e2) { }; \ - void show(std::ostream & str) \ - { \ - str << *e1 << " " s " " << *e2; \ - } \ - void bindVars(const StaticEnv & env) \ - { \ - e1->bindVars(env); e2->bindVars(env); \ - } \ - void eval(EvalState & state, Env & env, Value & v); \ - }; - -MakeBinOp(App, "") -MakeBinOp(OpEq, "==") -MakeBinOp(OpNEq, "!=") -MakeBinOp(OpAnd, "&&") -MakeBinOp(OpOr, "||") -MakeBinOp(OpImpl, "->") -MakeBinOp(OpUpdate, "//") -MakeBinOp(OpConcatLists, "++") - -struct ExprConcatStrings : Expr -{ - bool forceString; - vector<Expr *> * es; - ExprConcatStrings(bool forceString, vector<Expr *> * es) - : forceString(forceString), es(es) { }; - COMMON_METHODS -}; - -struct ExprPos : Expr -{ - Pos pos; - ExprPos(const Pos & pos) : pos(pos) { }; - COMMON_METHODS -}; - - -/* Static environments are used to map variable names onto (level, - displacement) pairs used to obtain the value of the variable at - runtime. */ -struct StaticEnv -{ - bool isWith; - const StaticEnv * up; - typedef std::map<Symbol, unsigned int> Vars; - Vars vars; - StaticEnv(bool isWith, const StaticEnv * up) : isWith(isWith), up(up) { }; -}; - - - -} diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y deleted file mode 100644 index ab0b862246..0000000000 --- a/src/libexpr/parser.y +++ /dev/null @@ -1,651 +0,0 @@ -%glr-parser -%pure-parser -%locations -%error-verbose -%defines -/* %no-lines */ -%parse-param { void * scanner } -%parse-param { nix::ParseData * data } -%lex-param { void * scanner } -%lex-param { nix::ParseData * data } -%expect 1 -%expect-rr 1 - -%code requires { - -#ifndef BISON_HEADER -#define BISON_HEADER - -#include "util.hh" - -#include "nixexpr.hh" -#include "eval.hh" - -namespace nix { - - struct ParseData - { - EvalState & state; - SymbolTable & symbols; - Expr * result; - Path basePath; - Symbol path; - string error; - Symbol sLetBody; - ParseData(EvalState & state) - : state(state) - , symbols(state.symbols) - , sLetBody(symbols.create("<let-body>")) - { }; - }; - -} - -#define YY_DECL int yylex \ - (YYSTYPE * yylval_param, YYLTYPE * yylloc_param, yyscan_t yyscanner, nix::ParseData * data) - -#endif - -} - -%{ - -#include "parser-tab.hh" -#include "lexer-tab.hh" - -#include <stdio.h> -#include <stdlib.h> -#include <string.h> - -YY_DECL; - -using namespace nix; - - -namespace nix { - - -static void dupAttr(const AttrPath & attrPath, const Pos & pos, const Pos & prevPos) -{ - throw ParseError(format("attribute `%1%' at %2% already defined at %3%") - % showAttrPath(attrPath) % pos % prevPos); -} - - -static void dupAttr(Symbol attr, const Pos & pos, const Pos & prevPos) -{ - throw ParseError(format("attribute `%1%' at %2% already defined at %3%") - % attr % pos % prevPos); -} - - -static void addAttr(ExprAttrs * attrs, AttrPath & attrPath, - Expr * e, const Pos & pos) -{ - AttrPath::iterator i; - // All attrpaths have at least one attr - assert(!attrPath.empty()); - for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) { - if (i->symbol.set()) { - ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); - if (j != attrs->attrs.end()) { - if (!j->second.inherited) { - ExprAttrs * attrs2 = dynamic_cast<ExprAttrs *>(j->second.e); - if (!attrs2) dupAttr(attrPath, pos, j->second.pos); - attrs = attrs2; - } else - dupAttr(attrPath, pos, j->second.pos); - } else { - ExprAttrs * nested = new ExprAttrs; - attrs->attrs[i->symbol] = ExprAttrs::AttrDef(nested, pos); - attrs = nested; - } - } else { - ExprAttrs *nested = new ExprAttrs; - attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, nested, pos)); - attrs = nested; - } - } - if (i->symbol.set()) { - ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); - if (j != attrs->attrs.end()) { - dupAttr(attrPath, pos, j->second.pos); - } else { - attrs->attrs[i->symbol] = ExprAttrs::AttrDef(e, pos); - e->setName(i->symbol); - } - } else { - attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, e, pos)); - } -} - - -static void addFormal(const Pos & pos, Formals * formals, const Formal & formal) -{ - if (formals->argNames.find(formal.name) != formals->argNames.end()) - throw ParseError(format("duplicate formal function argument `%1%' at %2%") - % formal.name % pos); - formals->formals.push_front(formal); - formals->argNames.insert(formal.name); -} - - -static Expr * stripIndentation(SymbolTable & symbols, vector<Expr *> & es) -{ - if (es.empty()) return new ExprString(symbols.create("")); - - /* Figure out the minimum indentation. Note that by design - whitespace-only final lines are not taken into account. (So - the " " in "\n ''" is ignored, but the " " in "\n foo''" is.) */ - bool atStartOfLine = true; /* = seen only whitespace in the current line */ - unsigned int minIndent = 1000000; - unsigned int curIndent = 0; - foreach (vector<Expr *>::iterator, i, es) { - ExprIndStr * e = dynamic_cast<ExprIndStr *>(*i); - if (!e) { - /* Anti-quotations end the current start-of-line whitespace. */ - if (atStartOfLine) { - atStartOfLine = false; - if (curIndent < minIndent) minIndent = curIndent; - } - continue; - } - for (unsigned int j = 0; j < e->s.size(); ++j) { - if (atStartOfLine) { - if (e->s[j] == ' ') - curIndent++; - else if (e->s[j] == '\n') { - /* Empty line, doesn't influence minimum - indentation. */ - curIndent = 0; - } else { - atStartOfLine = false; - if (curIndent < minIndent) minIndent = curIndent; - } - } else if (e->s[j] == '\n') { - atStartOfLine = true; - curIndent = 0; - } - } - } - - /* Strip spaces from each line. */ - vector<Expr *> * es2 = new vector<Expr *>; - atStartOfLine = true; - unsigned int curDropped = 0; - unsigned int n = es.size(); - for (vector<Expr *>::iterator i = es.begin(); i != es.end(); ++i, --n) { - ExprIndStr * e = dynamic_cast<ExprIndStr *>(*i); - if (!e) { - atStartOfLine = false; - curDropped = 0; - es2->push_back(*i); - continue; - } - - string s2; - for (unsigned int j = 0; j < e->s.size(); ++j) { - if (atStartOfLine) { - if (e->s[j] == ' ') { - if (curDropped++ >= minIndent) - s2 += e->s[j]; - } - else if (e->s[j] == '\n') { - curDropped = 0; - s2 += e->s[j]; - } else { - atStartOfLine = false; - curDropped = 0; - s2 += e->s[j]; - } - } else { - s2 += e->s[j]; - if (e->s[j] == '\n') atStartOfLine = true; - } - } - - /* Remove the last line if it is empty and consists only of - spaces. */ - if (n == 1) { - string::size_type p = s2.find_last_of('\n'); - if (p != string::npos && s2.find_first_not_of(' ', p + 1) == string::npos) - s2 = string(s2, 0, p + 1); - } - - es2->push_back(new ExprString(symbols.create(s2))); - } - - /* If this is a single string, then don't do a concatenation. */ - return es2->size() == 1 && dynamic_cast<ExprString *>((*es2)[0]) ? (*es2)[0] : new ExprConcatStrings(true, es2); -} - - -void backToString(yyscan_t scanner); -void backToIndString(yyscan_t scanner); - - -static Pos makeCurPos(const YYLTYPE & loc, ParseData * data) -{ - return Pos(data->path, loc.first_line, loc.first_column); -} - -#define CUR_POS makeCurPos(*yylocp, data) - - -} - - -void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * error) -{ - data->error = (format("%1%, at %2%") - % error % makeCurPos(*loc, data)).str(); -} - - -%} - -%union { - // !!! We're probably leaking stuff here. - nix::Expr * e; - nix::ExprList * list; - nix::ExprAttrs * attrs; - nix::Formals * formals; - nix::Formal * formal; - nix::NixInt n; - const char * id; // !!! -> Symbol - char * path; - char * uri; - std::vector<nix::AttrName> * attrNames; - std::vector<nix::Expr *> * string_parts; -} - -%type <e> start expr expr_function expr_if expr_op -%type <e> expr_app expr_select expr_simple -%type <list> expr_list -%type <attrs> binds -%type <formals> formals -%type <formal> formal -%type <attrNames> attrs attrpath -%type <string_parts> string_parts_interpolated ind_string_parts -%type <e> string_parts string_attr -%type <id> attr -%token <id> ID ATTRPATH -%token <e> STR IND_STR -%token <n> INT -%token <path> PATH SPATH -%token <uri> URI -%token IF THEN ELSE ASSERT WITH LET IN REC INHERIT EQ NEQ AND OR IMPL OR_KW -%token DOLLAR_CURLY /* == ${ */ -%token IND_STRING_OPEN IND_STRING_CLOSE -%token ELLIPSIS - -%nonassoc IMPL -%left OR -%left AND -%nonassoc EQ NEQ -%left '<' '>' LEQ GEQ -%right UPDATE -%left NOT -%left '+' '-' -%left '*' '/' -%right CONCAT -%nonassoc '?' -%nonassoc '~' -%nonassoc NEGATE - -%% - -start: expr { data->result = $1; }; - -expr: expr_function; - -expr_function - : ID ':' expr_function - { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), false, 0, $3); } - | '{' formals '}' ':' expr_function - { $$ = new ExprLambda(CUR_POS, data->symbols.create(""), true, $2, $5); } - | '{' formals '}' '@' ID ':' expr_function - { $$ = new ExprLambda(CUR_POS, data->symbols.create($5), true, $2, $7); } - | ID '@' '{' formals '}' ':' expr_function - { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), true, $4, $7); } - | ASSERT expr ';' expr_function - { $$ = new ExprAssert(CUR_POS, $2, $4); } - | WITH expr ';' expr_function - { $$ = new ExprWith(CUR_POS, $2, $4); } - | LET binds IN expr_function - { if (!$2->dynamicAttrs.empty()) - throw ParseError(format("dynamic attributes not allowed in let at %1%") - % CUR_POS); - $$ = new ExprLet($2, $4); - } - | expr_if - ; - -expr_if - : IF expr THEN expr ELSE expr { $$ = new ExprIf($2, $4, $6); } - | expr_op - ; - -expr_op - : '!' expr_op %prec NOT { $$ = new ExprOpNot($2); } -| '-' expr_op %prec NEGATE { $$ = new ExprApp(new ExprApp(new ExprBuiltin(data->symbols.create("sub")), new ExprInt(0)), $2); } - | expr_op EQ expr_op { $$ = new ExprOpEq($1, $3); } - | expr_op NEQ expr_op { $$ = new ExprOpNEq($1, $3); } - | expr_op '<' expr_op { $$ = new ExprApp(new ExprApp(new ExprBuiltin(data->symbols.create("lessThan")), $1), $3); } - | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprApp(new ExprApp(new ExprBuiltin(data->symbols.create("lessThan")), $3), $1)); } - | expr_op '>' expr_op { $$ = new ExprApp(new ExprApp(new ExprBuiltin(data->symbols.create("lessThan")), $3), $1); } - | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprApp(new ExprApp(new ExprBuiltin(data->symbols.create("lessThan")), $1), $3)); } - | expr_op AND expr_op { $$ = new ExprOpAnd($1, $3); } - | expr_op OR expr_op { $$ = new ExprOpOr($1, $3); } - | expr_op IMPL expr_op { $$ = new ExprOpImpl($1, $3); } - | expr_op UPDATE expr_op { $$ = new ExprOpUpdate($1, $3); } - | expr_op '?' attrpath { $$ = new ExprOpHasAttr($1, *$3); } - | expr_op '+' expr_op - { vector<Expr *> * l = new vector<Expr *>; - l->push_back($1); - l->push_back($3); - $$ = new ExprConcatStrings(false, l); - } - | expr_op '-' expr_op { $$ = new ExprApp(new ExprApp(new ExprBuiltin(data->symbols.create("sub")), $1), $3); } - | expr_op '*' expr_op { $$ = new ExprApp(new ExprApp(new ExprBuiltin(data->symbols.create("mul")), $1), $3); } - | expr_op '/' expr_op { $$ = new ExprApp(new ExprApp(new ExprBuiltin(data->symbols.create("div")), $1), $3); } - | expr_op CONCAT expr_op { $$ = new ExprOpConcatLists($1, $3); } - | expr_app - ; - -expr_app - : expr_app expr_select - { $$ = new ExprApp($1, $2); } - | expr_select { $$ = $1; } - ; - -expr_select - : expr_simple '.' attrpath - { $$ = new ExprSelect($1, *$3, 0); } - | expr_simple '.' attrpath OR_KW expr_select - { $$ = new ExprSelect($1, *$3, $5); } - | /* Backwards compatibility: because Nixpkgs has a rarely used - function named âorâ, allow stuff like âmap or [...]â. */ - expr_simple OR_KW - { $$ = new ExprApp($1, new ExprVar(CUR_POS, data->symbols.create("or"))); } - | expr_simple { $$ = $1; } - ; - -expr_simple - : ID { - if (strcmp($1, "__curPos") == 0) - $$ = new ExprPos(CUR_POS); - else - $$ = new ExprVar(CUR_POS, data->symbols.create($1)); - } - | INT { $$ = new ExprInt($1); } - | '"' string_parts '"' { $$ = $2; } - | IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE { - $$ = stripIndentation(data->symbols, *$2); - } - | PATH { $$ = new ExprPath(absPath($1, data->basePath)); } - | SPATH { - string path($1 + 1, strlen($1) - 2); - Path path2 = data->state.findFile(path); - /* The file wasn't found in the search path. However, we can't - throw an error here, because the expression might never be - evaluated. So return an expression that lazily calls - âthrowâ. */ - $$ = path2 == "" - ? (Expr * ) new ExprApp( - new ExprBuiltin(data->symbols.create("throw")), - new ExprString(data->symbols.create( - (format("file `%1%' was not found in the Nix search path (add it using $NIX_PATH or -I)") % path).str()))) - : (Expr * ) new ExprPath(path2); - } - | URI { $$ = new ExprString(data->symbols.create($1)); } - | '(' expr ')' { $$ = $2; } - /* Let expressions `let {..., body = ...}' are just desugared - into `(rec {..., body = ...}).body'. */ - | LET '{' binds '}' - { $3->recursive = true; $$ = new ExprSelect($3, data->symbols.create("body")); } - | REC '{' binds '}' - { $3->recursive = true; $$ = $3; } - | '{' binds '}' - { $$ = $2; } - | '[' expr_list ']' { $$ = $2; } - ; - -string_parts - : STR - | string_parts_interpolated { $$ = new ExprConcatStrings(true, $1); } - | { $$ = new ExprString(data->symbols.create("")); } - ; - -string_parts_interpolated - : string_parts_interpolated STR { $$ = $1; $1->push_back($2); } - | string_parts_interpolated DOLLAR_CURLY expr '}' { backToString(scanner); $$ = $1; $1->push_back($3); } - | STR DOLLAR_CURLY expr '}' - { - backToString(scanner); - $$ = new vector<Expr *>; - $$->push_back($1); - $$->push_back($3); - } - | DOLLAR_CURLY expr '}' - { - backToString(scanner); - $$ = new vector<Expr *>; - $$->push_back($2); - } - ; - -ind_string_parts - : ind_string_parts IND_STR { $$ = $1; $1->push_back($2); } - | ind_string_parts DOLLAR_CURLY expr '}' { backToIndString(scanner); $$ = $1; $1->push_back($3); } - | { $$ = new vector<Expr *>; } - ; - -binds - : binds attrpath '=' expr ';' { $$ = $1; addAttr($$, *$2, $4, makeCurPos(@2, data)); } - | binds INHERIT attrs ';' - { $$ = $1; - foreach (AttrPath::iterator, i, *$3) { - if ($$->attrs.find(i->symbol) != $$->attrs.end()) - dupAttr(i->symbol, makeCurPos(@3, data), $$->attrs[i->symbol].pos); - Pos pos = makeCurPos(@3, data); - $$->attrs[i->symbol] = ExprAttrs::AttrDef(new ExprVar(CUR_POS, i->symbol), pos, true); - } - } - | binds INHERIT '(' expr ')' attrs ';' - { $$ = $1; - /* !!! Should ensure sharing of the expression in $4. */ - foreach (AttrPath::iterator, i, *$6) { - if ($$->attrs.find(i->symbol) != $$->attrs.end()) - dupAttr(i->symbol, makeCurPos(@6, data), $$->attrs[i->symbol].pos); - $$->attrs[i->symbol] = ExprAttrs::AttrDef(new ExprSelect($4, i->symbol), makeCurPos(@6, data)); - } - } - | { $$ = new ExprAttrs; } - ; - -attrs - : attrs attr { $$ = $1; $1->push_back(AttrName(data->symbols.create($2))); } - | attrs string_attr - { $$ = $1; - ExprString *str = dynamic_cast<ExprString *>($2); - if (str) { - $$->push_back(AttrName(str->s)); - delete str; - } else - throw ParseError(format("dynamic attributes not allowed in inherit at %1%") - % makeCurPos(@2, data)); - } - | { $$ = new AttrPath; } - ; - -attrpath - : attrpath '.' attr { $$ = $1; $1->push_back(AttrName(data->symbols.create($3))); } - | attrpath '.' string_attr - { $$ = $1; - ExprString *str = dynamic_cast<ExprString *>($3); - if (str) { - $$->push_back(AttrName(str->s)); - delete str; - } else - $$->push_back(AttrName(static_cast<ExprConcatStrings *>($3))); - } - | attr { $$ = new vector<AttrName>; $$->push_back(AttrName(data->symbols.create($1))); } - | string_attr - { $$ = new vector<AttrName>; - ExprString *str = dynamic_cast<ExprString *>($1); - if (str) { - $$->push_back(AttrName(str->s)); - delete str; - } else - $$->push_back(AttrName(static_cast<ExprConcatStrings *>($1))); - } - ; - -attr - : ID { $$ = $1; } - | OR_KW { $$ = "or"; } - ; - -string_attr - : '"' string_parts '"' { $$ = $2; } - | DOLLAR_CURLY expr '}' { $$ = new ExprConcatStrings(true, new vector<Expr*>(1, $2)); } - ; - -expr_list - : expr_list expr_select { $$ = $1; $1->elems.push_back($2); /* !!! dangerous */ } - | { $$ = new ExprList; } - ; - -formals - : formal ',' formals - { $$ = $3; addFormal(CUR_POS, $$, *$1); } - | formal - { $$ = new Formals; addFormal(CUR_POS, $$, *$1); $$->ellipsis = false; } - | - { $$ = new Formals; $$->ellipsis = false; } - | ELLIPSIS - { $$ = new Formals; $$->ellipsis = true; } - ; - -formal - : ID { $$ = new Formal(data->symbols.create($1), 0); } - | ID '?' expr { $$ = new Formal(data->symbols.create($1), $3); } - ; - -%% - - -#include <sys/types.h> -#include <sys/stat.h> -#include <fcntl.h> -#include <unistd.h> - -#include <eval.hh> - - -namespace nix { - - -Expr * EvalState::parse(const char * text, - const Path & path, const Path & basePath, StaticEnv & staticEnv) -{ - yyscan_t scanner; - ParseData data(*this); - data.basePath = basePath; - data.path = data.symbols.create(path); - - yylex_init(&scanner); - yy_scan_string(text, scanner); - int res = yyparse(scanner, &data); - yylex_destroy(scanner); - - if (res) throw ParseError(data.error); - - data.result->bindVars(staticEnv); - - return data.result; -} - - -Path resolveExprPath(Path path) -{ - assert(path[0] == '/'); - - /* If `path' is a symlink, follow it. This is so that relative - path references work. */ - struct stat st; - while (true) { - if (lstat(path.c_str(), &st)) - throw SysError(format("getting status of `%1%'") % path); - if (!S_ISLNK(st.st_mode)) break; - path = absPath(readLink(path), dirOf(path)); - } - - /* If `path' refers to a directory, append `/default.nix'. */ - if (S_ISDIR(st.st_mode)) - path = canonPath(path + "/default.nix"); - - return path; -} - - -Expr * EvalState::parseExprFromFile(const Path & path) -{ - return parse(readFile(path).c_str(), path, dirOf(path), staticBaseEnv); -} - - -Expr * EvalState::parseExprFromString(const string & s, const Path & basePath, StaticEnv & staticEnv) -{ - return parse(s.c_str(), "(string)", basePath, staticEnv); -} - - -Expr * EvalState::parseExprFromString(const string & s, const Path & basePath) -{ - return parseExprFromString(s, basePath, staticBaseEnv); -} - - - void EvalState::addToSearchPath(const string & s, bool warn) -{ - size_t pos = s.find('='); - string prefix; - Path path; - if (pos == string::npos) { - path = s; - } else { - prefix = string(s, 0, pos); - path = string(s, pos + 1); - } - - path = absPath(path); - if (pathExists(path)) { - debug(format("adding path `%1%' to the search path") % path); - searchPath.insert(searchPathInsertionPoint, std::pair<string, Path>(prefix, path)); - } else if (warn) - printMsg(lvlError, format("warning: Nix search path entry `%1%' does not exist, ignoring") % path); -} - - -Path EvalState::findFile(const string & path) -{ - foreach (SearchPath::iterator, i, searchPath) { - Path res; - if (i->first.empty()) - res = i->second + "/" + path; - else { - if (path.compare(0, i->first.size(), i->first) != 0 || - (path.size() > i->first.size() && path[i->first.size()] != '/')) - continue; - res = i->second + - (path.size() == i->first.size() ? "" : "/" + string(path, i->first.size())); - } - if (pathExists(res)) return canonPath(res); - } - return ""; -} - - -} diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc deleted file mode 100644 index ca316f08af..0000000000 --- a/src/libexpr/primops.cc +++ /dev/null @@ -1,1336 +0,0 @@ -#include "eval.hh" -#include "misc.hh" -#include "globals.hh" -#include "store-api.hh" -#include "util.hh" -#include "archive.hh" -#include "value-to-xml.hh" -#include "value-to-json.hh" -#include "names.hh" -#include "eval-inline.hh" - -#include <sys/types.h> -#include <sys/stat.h> -#include <unistd.h> - -#include <algorithm> -#include <cstring> - - -namespace nix { - - -/************************************************************* - * Miscellaneous - *************************************************************/ - - -/* Decode a context string â!<name>!<path>â into a pair <path, - name>. */ -std::pair<string, string> decodeContext(const string & s) -{ - if (s.at(0) == '!') { - size_t index = s.find("!", 1); - return std::pair<string, string>(string(s, index + 1), string(s, 1, index - 1)); - } else - return std::pair<string, string>(s, ""); -} - - -/* Load and evaluate an expression from path specified by the - argument. */ -static void prim_import(EvalState & state, Value * * args, Value & v) -{ - PathSet context; - Path path = state.coerceToPath(*args[0], context); - - foreach (PathSet::iterator, i, context) { - Path ctx = decodeContext(*i).first; - assert(isStorePath(ctx)); - if (!store->isValidPath(ctx)) - throw EvalError(format("cannot import `%1%', since path `%2%' is not valid") - % path % ctx); - if (isDerivation(ctx)) - try { - /* For performance, prefetch all substitute info. */ - PathSet willBuild, willSubstitute, unknown; - unsigned long long downloadSize, narSize; - queryMissing(*store, singleton<PathSet>(ctx), - willBuild, willSubstitute, unknown, downloadSize, narSize); - - /* !!! If using a substitute, we only need to fetch - the selected output of this derivation. */ - store->buildPaths(singleton<PathSet>(ctx)); - } catch (Error & e) { - throw ImportError(e.msg()); - } - } - - if (isStorePath(path) && store->isValidPath(path) && isDerivation(path)) { - Derivation drv = parseDerivation(readFile(path)); - Value & w = *state.allocValue(); - state.mkAttrs(w, 1 + drv.outputs.size()); - mkString(*state.allocAttr(w, state.sDrvPath), path, singleton<PathSet>("=" + path)); - state.mkList(*state.allocAttr(w, state.symbols.create("outputs")), drv.outputs.size()); - unsigned int outputs_index = 0; - - Value * outputsVal = w.attrs->find(state.symbols.create("outputs"))->value; - foreach (DerivationOutputs::iterator, i, drv.outputs) { - mkString(*state.allocAttr(w, state.symbols.create(i->first)), - i->second.path, singleton<PathSet>("!" + i->first + "!" + path)); - mkString(*(outputsVal->list.elems[outputs_index++] = state.allocValue()), - i->first); - } - w.attrs->sort(); - Value fun; - state.evalFile(state.findFile("nix/imported-drv-to-derivation.nix"), fun); - state.forceFunction(fun); - mkApp(v, fun, w); - state.forceAttrs(v); - } else { - state.evalFile(path, v); - } -} - - -/* Return a string representing the type of the expression. */ -static void prim_typeOf(EvalState & state, Value * * args, Value & v) -{ - state.forceValue(*args[0]); - string t; - switch (args[0]->type) { - case tInt: t = "int"; break; - case tBool: t = "bool"; break; - case tString: t = "string"; break; - case tPath: t = "path"; break; - case tNull: t = "null"; break; - case tAttrs: t = "set"; break; - case tList: t = "list"; break; - case tLambda: - case tPrimOp: - case tPrimOpApp: - t = "lambda"; - break; - default: abort(); - } - mkString(v, state.symbols.create(t)); -} - - -/* Determine whether the argument is the null value. */ -static void prim_isNull(EvalState & state, Value * * args, Value & v) -{ - state.forceValue(*args[0]); - mkBool(v, args[0]->type == tNull); -} - - -/* Determine whether the argument is a function. */ -static void prim_isFunction(EvalState & state, Value * * args, Value & v) -{ - state.forceValue(*args[0]); - mkBool(v, args[0]->type == tLambda); -} - - -/* Determine whether the argument is an integer. */ -static void prim_isInt(EvalState & state, Value * * args, Value & v) -{ - state.forceValue(*args[0]); - mkBool(v, args[0]->type == tInt); -} - - -/* Determine whether the argument is a string. */ -static void prim_isString(EvalState & state, Value * * args, Value & v) -{ - state.forceValue(*args[0]); - mkBool(v, args[0]->type == tString); -} - - -/* Determine whether the argument is a Boolean. */ -static void prim_isBool(EvalState & state, Value * * args, Value & v) -{ - state.forceValue(*args[0]); - mkBool(v, args[0]->type == tBool); -} - - -struct CompareValues -{ - bool operator () (const Value * v1, const Value * v2) const - { - if (v1->type != v2->type) - throw EvalError("cannot compare values of different types"); - switch (v1->type) { - case tInt: - return v1->integer < v2->integer; - case tString: - return strcmp(v1->string.s, v2->string.s) < 0; - case tPath: - return strcmp(v1->path, v2->path) < 0; - default: - throw EvalError(format("cannot compare %1% with %2%") % showType(*v1) % showType(*v2)); - } - } -}; - - -#if HAVE_BOEHMGC -typedef list<Value *, gc_allocator<Value *> > ValueList; -#else -typedef list<Value *> ValueList; -#endif - - -static void prim_genericClosure(EvalState & state, Value * * args, Value & v) -{ - startNest(nest, lvlDebug, "finding dependencies"); - - state.forceAttrs(*args[0]); - - /* Get the start set. */ - Bindings::iterator startSet = - args[0]->attrs->find(state.symbols.create("startSet")); - if (startSet == args[0]->attrs->end()) - throw EvalError("attribute `startSet' required"); - state.forceList(*startSet->value); - - ValueList workSet; - for (unsigned int n = 0; n < startSet->value->list.length; ++n) - workSet.push_back(startSet->value->list.elems[n]); - - /* Get the operator. */ - Bindings::iterator op = - args[0]->attrs->find(state.symbols.create("operator")); - if (op == args[0]->attrs->end()) - throw EvalError("attribute `operator' required"); - state.forceValue(*op->value); - - /* Construct the closure by applying the operator to element of - `workSet', adding the result to `workSet', continuing until - no new elements are found. */ - ValueList res; - // `doneKeys' doesn't need to be a GC root, because its values are - // reachable from res. - set<Value *, CompareValues> doneKeys; - while (!workSet.empty()) { - Value * e = *(workSet.begin()); - workSet.pop_front(); - - state.forceAttrs(*e); - - Bindings::iterator key = - e->attrs->find(state.symbols.create("key")); - if (key == e->attrs->end()) - throw EvalError("attribute `key' required"); - state.forceValue(*key->value); - - if (doneKeys.find(key->value) != doneKeys.end()) continue; - doneKeys.insert(key->value); - res.push_back(e); - - /* Call the `operator' function with `e' as argument. */ - Value call; - mkApp(call, *op->value, *e); - state.forceList(call); - - /* Add the values returned by the operator to the work set. */ - for (unsigned int n = 0; n < call.list.length; ++n) { - state.forceValue(*call.list.elems[n]); - workSet.push_back(call.list.elems[n]); - } - } - - /* Create the result list. */ - state.mkList(v, res.size()); - unsigned int n = 0; - foreach (ValueList::iterator, i, res) - v.list.elems[n++] = *i; -} - - -static void prim_abort(EvalState & state, Value * * args, Value & v) -{ - PathSet context; - throw Abort(format("evaluation aborted with the following error message: `%1%'") % - state.coerceToString(*args[0], context)); -} - - -static void prim_throw(EvalState & state, Value * * args, Value & v) -{ - PathSet context; - throw ThrownError(format("%1%") % state.coerceToString(*args[0], context)); -} - - -static void prim_addErrorContext(EvalState & state, Value * * args, Value & v) -{ - try { - state.forceValue(*args[1]); - v = *args[1]; - } catch (Error & e) { - PathSet context; - e.addPrefix(format("%1%\n") % state.coerceToString(*args[0], context)); - throw; - } -} - - -/* Try evaluating the argument. Success => {success=true; value=something;}, - * else => {success=false; value=false;} */ -static void prim_tryEval(EvalState & state, Value * * args, Value & v) -{ - state.mkAttrs(v, 2); - try { - state.forceValue(*args[0]); - v.attrs->push_back(Attr(state.sValue, args[0])); - mkBool(*state.allocAttr(v, state.symbols.create("success")), true); - } catch (AssertionError & e) { - mkBool(*state.allocAttr(v, state.sValue), false); - mkBool(*state.allocAttr(v, state.symbols.create("success")), false); - } - v.attrs->sort(); -} - - -/* Return an environment variable. Use with care. */ -static void prim_getEnv(EvalState & state, Value * * args, Value & v) -{ - string name = state.forceStringNoCtx(*args[0]); - mkString(v, getEnv(name)); -} - - -/* Evaluate the first expression and print it on standard error. Then - return the second expression. Useful for debugging. */ -static void prim_trace(EvalState & state, Value * * args, Value & v) -{ - state.forceValue(*args[0]); - if (args[0]->type == tString) - printMsg(lvlError, format("trace: %1%") % args[0]->string.s); - else - printMsg(lvlError, format("trace: %1%") % *args[0]); - state.forceValue(*args[1]); - v = *args[1]; -} - - -/************************************************************* - * Derivations - *************************************************************/ - - -/* Construct (as a unobservable side effect) a Nix derivation - expression that performs the derivation described by the argument - set. Returns the original set extended with the following - attributes: `outPath' containing the primary output path of the - derivation; `drvPath' containing the path of the Nix expression; - and `type' set to `derivation' to indicate that this is a - derivation. */ -static void prim_derivationStrict(EvalState & state, Value * * args, Value & v) -{ - startNest(nest, lvlVomit, "evaluating derivation"); - - state.forceAttrs(*args[0]); - - /* Figure out the name first (for stack backtraces). */ - Bindings::iterator attr = args[0]->attrs->find(state.sName); - if (attr == args[0]->attrs->end()) - throw EvalError("required attribute `name' missing"); - string drvName; - Pos & posDrvName(*attr->pos); - try { - drvName = state.forceStringNoCtx(*attr->value); - } catch (Error & e) { - e.addPrefix(format("while evaluating the derivation attribute `name' at %1%:\n") % posDrvName); - throw; - } - - /* Check whether null attributes should be ignored. */ - bool ignoreNulls = false; - attr = args[0]->attrs->find(state.sIgnoreNulls); - if (attr != args[0]->attrs->end()) - ignoreNulls = state.forceBool(*attr->value); - - /* Build the derivation expression by processing the attributes. */ - Derivation drv; - - PathSet context; - - string outputHash, outputHashAlgo; - bool outputHashRecursive = false; - - StringSet outputs; - outputs.insert("out"); - - foreach (Bindings::iterator, i, *args[0]->attrs) { - if (i->name == state.sIgnoreNulls) continue; - string key = i->name; - startNest(nest, lvlVomit, format("processing attribute `%1%'") % key); - - try { - - if (ignoreNulls) { - state.forceValue(*i->value); - if (i->value->type == tNull) continue; - } - - /* The `args' attribute is special: it supplies the - command-line arguments to the builder. */ - if (key == "args") { - state.forceList(*i->value); - for (unsigned int n = 0; n < i->value->list.length; ++n) { - string s = state.coerceToString(*i->value->list.elems[n], context, true); - drv.args.push_back(s); - } - } - - /* All other attributes are passed to the builder through - the environment. */ - else { - string s = state.coerceToString(*i->value, context, true); - drv.env[key] = s; - if (key == "builder") drv.builder = s; - else if (i->name == state.sSystem) drv.platform = s; - else if (i->name == state.sName) { - drvName = s; - printMsg(lvlVomit, format("derivation name is `%1%'") % drvName); - } - else if (key == "outputHash") outputHash = s; - else if (key == "outputHashAlgo") outputHashAlgo = s; - else if (key == "outputHashMode") { - if (s == "recursive") outputHashRecursive = true; - else if (s == "flat") outputHashRecursive = false; - else throw EvalError(format("invalid value `%1%' for `outputHashMode' attribute") % s); - } - else if (key == "outputs") { - Strings tmp = tokenizeString<Strings>(s); - outputs.clear(); - foreach (Strings::iterator, j, tmp) { - if (outputs.find(*j) != outputs.end()) - throw EvalError(format("duplicate derivation output `%1%'") % *j); - /* !!! Check whether *j is a valid attribute - name. */ - /* Derivations cannot be named âdrvâ, because - then we'd have an attribute âdrvPathâ in - the resulting set. */ - if (*j == "drv") - throw EvalError(format("invalid derivation output name `drv'") % *j); - outputs.insert(*j); - } - if (outputs.empty()) - throw EvalError("derivation cannot have an empty set of outputs"); - } - } - - } catch (Error & e) { - e.addPrefix(format("while evaluating the attribute `%1%' of the derivation `%2%' at %3%:\n") - % key % drvName % posDrvName); - throw; - } - } - - /* Everything in the context of the strings in the derivation - attributes should be added as dependencies of the resulting - derivation. */ - foreach (PathSet::iterator, i, context) { - Path path = *i; - - /* Paths marked with `=' denote that the path of a derivation - is explicitly passed to the builder. Since that allows the - builder to gain access to every path in the dependency - graph of the derivation (including all outputs), all paths - in the graph must be added to this derivation's list of - inputs to ensure that they are available when the builder - runs. */ - if (path.at(0) == '=') { - /* !!! This doesn't work if readOnlyMode is set. */ - PathSet refs; computeFSClosure(*store, string(path, 1), refs); - foreach (PathSet::iterator, j, refs) { - drv.inputSrcs.insert(*j); - if (isDerivation(*j)) - drv.inputDrvs[*j] = store->queryDerivationOutputNames(*j); - } - } - - /* See prim_unsafeDiscardOutputDependency. */ - else if (path.at(0) == '~') - drv.inputSrcs.insert(string(path, 1)); - - /* Handle derivation outputs of the form â!<name>!<path>â. */ - else if (path.at(0) == '!') { - std::pair<string, string> ctx = decodeContext(path); - drv.inputDrvs[ctx.first].insert(ctx.second); - } - - /* Handle derivation contexts returned by - âbuiltins.storePathâ. */ - else if (isDerivation(path)) - drv.inputDrvs[path] = store->queryDerivationOutputNames(path); - - /* Otherwise it's a source file. */ - else - drv.inputSrcs.insert(path); - } - - /* Do we have all required attributes? */ - if (drv.builder == "") - throw EvalError("required attribute `builder' missing"); - if (drv.platform == "") - throw EvalError("required attribute `system' missing"); - - /* Check whether the derivation name is valid. */ - checkStoreName(drvName); - if (isDerivation(drvName)) - throw EvalError(format("derivation names are not allowed to end in `%1%'") - % drvExtension); - - if (outputHash != "") { - /* Handle fixed-output derivations. */ - if (outputs.size() != 1 || *(outputs.begin()) != "out") - throw Error("multiple outputs are not supported in fixed-output derivations"); - - HashType ht = parseHashType(outputHashAlgo); - if (ht == htUnknown) - throw EvalError(format("unknown hash algorithm `%1%'") % outputHashAlgo); - Hash h = parseHash16or32(ht, outputHash); - outputHash = printHash(h); - if (outputHashRecursive) outputHashAlgo = "r:" + outputHashAlgo; - - Path outPath = makeFixedOutputPath(outputHashRecursive, ht, h, drvName); - drv.env["out"] = outPath; - drv.outputs["out"] = DerivationOutput(outPath, outputHashAlgo, outputHash); - } - - else { - /* Construct the "masked" store derivation, which is the final - one except that in the list of outputs, the output paths - are empty, and the corresponding environment variables have - an empty value. This ensures that changes in the set of - output names do get reflected in the hash. */ - foreach (StringSet::iterator, i, outputs) { - drv.env[*i] = ""; - drv.outputs[*i] = DerivationOutput("", "", ""); - } - - /* Use the masked derivation expression to compute the output - path. */ - Hash h = hashDerivationModulo(*store, drv); - - foreach (DerivationOutputs::iterator, i, drv.outputs) - if (i->second.path == "") { - Path outPath = makeOutputPath(i->first, h, drvName); - drv.env[i->first] = outPath; - i->second.path = outPath; - } - } - - /* Write the resulting term into the Nix store directory. */ - Path drvPath = writeDerivation(*store, drv, drvName, state.repair); - - printMsg(lvlChatty, format("instantiated `%1%' -> `%2%'") - % drvName % drvPath); - - /* Optimisation, but required in read-only mode! because in that - case we don't actually write store derivations, so we can't - read them later. */ - drvHashes[drvPath] = hashDerivationModulo(*store, drv); - - state.mkAttrs(v, 1 + drv.outputs.size()); - mkString(*state.allocAttr(v, state.sDrvPath), drvPath, singleton<PathSet>("=" + drvPath)); - foreach (DerivationOutputs::iterator, i, drv.outputs) { - mkString(*state.allocAttr(v, state.symbols.create(i->first)), - i->second.path, singleton<PathSet>("!" + i->first + "!" + drvPath)); - } - v.attrs->sort(); -} - - -/************************************************************* - * Paths - *************************************************************/ - - -/* Convert the argument to a path. !!! obsolete? */ -static void prim_toPath(EvalState & state, Value * * args, Value & v) -{ - PathSet context; - Path path = state.coerceToPath(*args[0], context); - mkString(v, canonPath(path), context); -} - - -/* Allow a valid store path to be used in an expression. This is - useful in some generated expressions such as in nix-push, which - generates a call to a function with an already existing store path - as argument. You don't want to use `toPath' here because it copies - the path to the Nix store, which yields a copy like - /nix/store/newhash-oldhash-oldname. In the past, `toPath' had - special case behaviour for store paths, but that created weird - corner cases. */ -static void prim_storePath(EvalState & state, Value * * args, Value & v) -{ - PathSet context; - Path path = state.coerceToPath(*args[0], context); - /* Resolve symlinks in âpathâ, unless âpathâ itself is a symlink - directly in the store. The latter condition is necessary so - e.g. nix-push does the right thing. */ - if (!isStorePath(path)) path = canonPath(path, true); - if (!isInStore(path)) - throw EvalError(format("path `%1%' is not in the Nix store") % path); - Path path2 = toStorePath(path); - if (!settings.readOnlyMode) - store->ensurePath(path2); - context.insert(path2); - mkString(v, path, context); -} - - -static void prim_pathExists(EvalState & state, Value * * args, Value & v) -{ - PathSet context; - Path path = state.coerceToPath(*args[0], context); - if (!context.empty()) - throw EvalError(format("string `%1%' cannot refer to other paths") % path); - mkBool(v, pathExists(path)); -} - - -/* Return the base name of the given string, i.e., everything - following the last slash. */ -static void prim_baseNameOf(EvalState & state, Value * * args, Value & v) -{ - PathSet context; - mkString(v, baseNameOf(state.coerceToString(*args[0], context)), context); -} - - -/* Return the directory of the given path, i.e., everything before the - last slash. Return either a path or a string depending on the type - of the argument. */ -static void prim_dirOf(EvalState & state, Value * * args, Value & v) -{ - PathSet context; - Path dir = dirOf(state.coerceToPath(*args[0], context)); - if (args[0]->type == tPath) mkPath(v, dir.c_str()); else mkString(v, dir, context); -} - - -/* Return the contents of a file as a string. */ -static void prim_readFile(EvalState & state, Value * * args, Value & v) -{ - PathSet context; - Path path = state.coerceToPath(*args[0], context); - if (!context.empty()) - throw EvalError(format("string `%1%' cannot refer to other paths") % path); - mkString(v, readFile(path).c_str()); -} - - -/************************************************************* - * Creating files - *************************************************************/ - - -/* Convert the argument (which can be any Nix expression) to an XML - representation returned in a string. Not all Nix expressions can - be sensibly or completely represented (e.g., functions). */ -static void prim_toXML(EvalState & state, Value * * args, Value & v) -{ - std::ostringstream out; - PathSet context; - printValueAsXML(state, true, false, *args[0], out, context); - mkString(v, out.str(), context); -} - - -/* Convert the argument (which can be any Nix expression) to a JSON - string. Not all Nix expressions can be sensibly or completely - represented (e.g., functions). */ -static void prim_toJSON(EvalState & state, Value * * args, Value & v) -{ - std::ostringstream out; - PathSet context; - printValueAsJSON(state, true, *args[0], out, context); - mkString(v, out.str(), context); -} - - -/* Store a string in the Nix store as a source file that can be used - as an input by derivations. */ -static void prim_toFile(EvalState & state, Value * * args, Value & v) -{ - PathSet context; - string name = state.forceStringNoCtx(*args[0]); - string contents = state.forceString(*args[1], context); - - PathSet refs; - - foreach (PathSet::iterator, i, context) { - Path path = *i; - if (path.at(0) == '=') path = string(path, 1); - if (isDerivation(path)) - throw EvalError(format("in `toFile': the file `%1%' cannot refer to derivation outputs") % name); - refs.insert(path); - } - - Path storePath = settings.readOnlyMode - ? computeStorePathForText(name, contents, refs) - : store->addTextToStore(name, contents, refs, state.repair); - - /* Note: we don't need to add `context' to the context of the - result, since `storePath' itself has references to the paths - used in args[1]. */ - - mkString(v, storePath, singleton<PathSet>(storePath)); -} - - -struct FilterFromExpr : PathFilter -{ - EvalState & state; - Value & filter; - - FilterFromExpr(EvalState & state, Value & filter) - : state(state), filter(filter) - { - } - - bool operator () (const Path & path) - { - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path `%1%'") % path); - - /* Call the filter function. The first argument is the path, - the second is a string indicating the type of the file. */ - Value arg1; - mkString(arg1, path); - - Value fun2; - state.callFunction(filter, arg1, fun2); - - Value arg2; - mkString(arg2, - S_ISREG(st.st_mode) ? "regular" : - S_ISDIR(st.st_mode) ? "directory" : - S_ISLNK(st.st_mode) ? "symlink" : - "unknown" /* not supported, will fail! */); - - Value res; - state.callFunction(fun2, arg2, res); - - return state.forceBool(res); - } -}; - - -static void prim_filterSource(EvalState & state, Value * * args, Value & v) -{ - PathSet context; - Path path = state.coerceToPath(*args[1], context); - if (!context.empty()) - throw EvalError(format("string `%1%' cannot refer to other paths") % path); - - state.forceValue(*args[0]); - if (args[0]->type != tLambda) - throw TypeError(format("first argument in call to `filterSource' is not a function but %1%") % showType(*args[0])); - - FilterFromExpr filter(state, *args[0]); - - Path dstPath = settings.readOnlyMode - ? computeStorePathForPath(path, true, htSHA256, filter).first - : store->addToStore(path, true, htSHA256, filter, state.repair); - - mkString(v, dstPath, singleton<PathSet>(dstPath)); -} - - -/************************************************************* - * Sets - *************************************************************/ - - -/* Return the names of the attributes in a set as a sorted list of - strings. */ -static void prim_attrNames(EvalState & state, Value * * args, Value & v) -{ - state.forceAttrs(*args[0]); - - state.mkList(v, args[0]->attrs->size()); - - StringSet names; - foreach (Bindings::iterator, i, *args[0]->attrs) - names.insert(i->name); - - unsigned int n = 0; - foreach (StringSet::iterator, i, names) - mkString(*(v.list.elems[n++] = state.allocValue()), *i); -} - - -/* Dynamic version of the `.' operator. */ -void prim_getAttr(EvalState & state, Value * * args, Value & v) -{ - string attr = state.forceStringNoCtx(*args[0]); - state.forceAttrs(*args[1]); - // !!! Should we create a symbol here or just do a lookup? - Bindings::iterator i = args[1]->attrs->find(state.symbols.create(attr)); - if (i == args[1]->attrs->end()) - throw EvalError(format("attribute `%1%' missing") % attr); - // !!! add to stack trace? - if (state.countCalls && i->pos) state.attrSelects[*i->pos]++; - state.forceValue(*i->value); - v = *i->value; -} - - -/* Return position information of the specified attribute. */ -void prim_unsafeGetAttrPos(EvalState & state, Value * * args, Value & v) -{ - string attr = state.forceStringNoCtx(*args[0]); - state.forceAttrs(*args[1]); - Bindings::iterator i = args[1]->attrs->find(state.symbols.create(attr)); - if (i == args[1]->attrs->end()) - mkNull(v); - else - state.mkPos(v, i->pos); -} - - -/* Dynamic version of the `?' operator. */ -static void prim_hasAttr(EvalState & state, Value * * args, Value & v) -{ - string attr = state.forceStringNoCtx(*args[0]); - state.forceAttrs(*args[1]); - mkBool(v, args[1]->attrs->find(state.symbols.create(attr)) != args[1]->attrs->end()); -} - - -/* Determine whether the argument is a set. */ -static void prim_isAttrs(EvalState & state, Value * * args, Value & v) -{ - state.forceValue(*args[0]); - mkBool(v, args[0]->type == tAttrs); -} - - -static void prim_removeAttrs(EvalState & state, Value * * args, Value & v) -{ - state.forceAttrs(*args[0]); - state.forceList(*args[1]); - - /* Get the attribute names to be removed. */ - std::set<Symbol> names; - for (unsigned int i = 0; i < args[1]->list.length; ++i) { - state.forceStringNoCtx(*args[1]->list.elems[i]); - names.insert(state.symbols.create(args[1]->list.elems[i]->string.s)); - } - - /* Copy all attributes not in that set. Note that we don't need - to sort v.attrs because it's a subset of an already sorted - vector. */ - state.mkAttrs(v, args[0]->attrs->size()); - foreach (Bindings::iterator, i, *args[0]->attrs) { - if (names.find(i->name) == names.end()) - v.attrs->push_back(*i); - } -} - - -/* Builds a set from a list specifying (name, value) pairs. To be - precise, a list [{name = "name1"; value = value1;} ... {name = - "nameN"; value = valueN;}] is transformed to {name1 = value1; - ... nameN = valueN;}. In case of duplicate occurences of the same - name, the first takes precedence. */ -static void prim_listToAttrs(EvalState & state, Value * * args, Value & v) -{ - state.forceList(*args[0]); - - state.mkAttrs(v, args[0]->list.length); - - std::set<Symbol> seen; - - for (unsigned int i = 0; i < args[0]->list.length; ++i) { - Value & v2(*args[0]->list.elems[i]); - state.forceAttrs(v2); - - Bindings::iterator j = v2.attrs->find(state.sName); - if (j == v2.attrs->end()) - throw TypeError("`name' attribute missing in a call to `listToAttrs'"); - string name = state.forceStringNoCtx(*j->value); - - Symbol sym = state.symbols.create(name); - if (seen.find(sym) == seen.end()) { - Bindings::iterator j2 = v2.attrs->find(state.symbols.create(state.sValue)); - if (j2 == v2.attrs->end()) - throw TypeError("`value' attribute missing in a call to `listToAttrs'"); - - v.attrs->push_back(Attr(sym, j2->value, j2->pos)); - seen.insert(sym); - } - } - - v.attrs->sort(); -} - - -/* Return the right-biased intersection of two sets as1 and as2, - i.e. a set that contains every attribute from as2 that is also a - member of as1. */ -static void prim_intersectAttrs(EvalState & state, Value * * args, Value & v) -{ - state.forceAttrs(*args[0]); - state.forceAttrs(*args[1]); - - state.mkAttrs(v, std::min(args[0]->attrs->size(), args[1]->attrs->size())); - - foreach (Bindings::iterator, i, *args[0]->attrs) { - Bindings::iterator j = args[1]->attrs->find(i->name); - if (j != args[1]->attrs->end()) - v.attrs->push_back(*j); - } -} - - -/* Return a set containing the names of the formal arguments expected - by the function `f'. The value of each attribute is a Boolean - denoting whether has a default value. For instance, - - functionArgs ({ x, y ? 123}: ...) - => { x = false; y = true; } - - "Formal argument" here refers to the attributes pattern-matched by - the function. Plain lambdas are not included, e.g. - - functionArgs (x: ...) - => { } -*/ -static void prim_functionArgs(EvalState & state, Value * * args, Value & v) -{ - state.forceValue(*args[0]); - if (args[0]->type != tLambda) - throw TypeError("`functionArgs' requires a function"); - - if (!args[0]->lambda.fun->matchAttrs) { - state.mkAttrs(v, 0); - return; - } - - state.mkAttrs(v, args[0]->lambda.fun->formals->formals.size()); - foreach (Formals::Formals_::iterator, i, args[0]->lambda.fun->formals->formals) - // !!! should optimise booleans (allocate only once) - mkBool(*state.allocAttr(v, i->name), i->def); - v.attrs->sort(); -} - - -/************************************************************* - * Lists - *************************************************************/ - - -/* Determine whether the argument is a list. */ -static void prim_isList(EvalState & state, Value * * args, Value & v) -{ - state.forceValue(*args[0]); - mkBool(v, args[0]->type == tList); -} - - -static void elemAt(EvalState & state, Value & list, int n, Value & v) -{ - state.forceList(list); - if (n < 0 || n >= list.list.length) - throw Error(format("list index %1% is out of bounds") % n); - state.forceValue(*list.list.elems[n]); - v = *list.list.elems[n]; -} - - -/* Return the n-1'th element of a list. */ -static void prim_elemAt(EvalState & state, Value * * args, Value & v) -{ - elemAt(state, *args[0], state.forceInt(*args[1]), v); -} - - -/* Return the first element of a list. */ -static void prim_head(EvalState & state, Value * * args, Value & v) -{ - elemAt(state, *args[0], 0, v); -} - - -/* Return a list consisting of everything but the the first element of - a list. Warning: this function takes O(n) time, so you probably - don't want to use it! */ -static void prim_tail(EvalState & state, Value * * args, Value & v) -{ - state.forceList(*args[0]); - if (args[0]->list.length == 0) - throw Error("`tail' called on an empty list"); - state.mkList(v, args[0]->list.length - 1); - for (unsigned int n = 0; n < v.list.length; ++n) - v.list.elems[n] = args[0]->list.elems[n + 1]; -} - - -/* Apply a function to every element of a list. */ -static void prim_map(EvalState & state, Value * * args, Value & v) -{ - state.forceFunction(*args[0]); - state.forceList(*args[1]); - - state.mkList(v, args[1]->list.length); - - for (unsigned int n = 0; n < v.list.length; ++n) - mkApp(*(v.list.elems[n] = state.allocValue()), - *args[0], *args[1]->list.elems[n]); -} - - -/* Filter a list using a predicate; that is, return a list containing - every element from the list for which the predicate function - returns true. */ -static void prim_filter(EvalState & state, Value * * args, Value & v) -{ - state.forceFunction(*args[0]); - state.forceList(*args[1]); - - // FIXME: putting this on the stack is risky. - Value * vs[args[1]->list.length]; - unsigned int k = 0; - - bool same = true; - for (unsigned int n = 0; n < args[1]->list.length; ++n) { - Value res; - state.callFunction(*args[0], *args[1]->list.elems[n], res); - if (state.forceBool(res)) - vs[k++] = args[1]->list.elems[n]; - else - same = false; - } - - if (same) - v = *args[1]; - else { - state.mkList(v, k); - for (unsigned int n = 0; n < k; ++n) v.list.elems[n] = vs[n]; - } -} - - -/* Return true if a list contains a given element. */ -static void prim_elem(EvalState & state, Value * * args, Value & v) -{ - bool res = false; - state.forceList(*args[1]); - for (unsigned int n = 0; n < args[1]->list.length; ++n) - if (state.eqValues(*args[0], *args[1]->list.elems[n])) { - res = true; - break; - } - mkBool(v, res); -} - - -/* Concatenate a list of lists. */ -static void prim_concatLists(EvalState & state, Value * * args, Value & v) -{ - state.forceList(*args[0]); - state.concatLists(v, args[0]->list.length, args[0]->list.elems); -} - - -/* Return the length of a list. This is an O(1) time operation. */ -static void prim_length(EvalState & state, Value * * args, Value & v) -{ - state.forceList(*args[0]); - mkInt(v, args[0]->list.length); -} - - -/************************************************************* - * Integer arithmetic - *************************************************************/ - - -static void prim_add(EvalState & state, Value * * args, Value & v) -{ - mkInt(v, state.forceInt(*args[0]) + state.forceInt(*args[1])); -} - - -static void prim_sub(EvalState & state, Value * * args, Value & v) -{ - mkInt(v, state.forceInt(*args[0]) - state.forceInt(*args[1])); -} - - -static void prim_mul(EvalState & state, Value * * args, Value & v) -{ - mkInt(v, state.forceInt(*args[0]) * state.forceInt(*args[1])); -} - - -static void prim_div(EvalState & state, Value * * args, Value & v) -{ - NixInt i2 = state.forceInt(*args[1]); - if (i2 == 0) throw EvalError("division by zero"); - mkInt(v, state.forceInt(*args[0]) / i2); -} - - -static void prim_lessThan(EvalState & state, Value * * args, Value & v) -{ - state.forceValue(*args[0]); - state.forceValue(*args[1]); - CompareValues comp; - mkBool(v, comp(args[0], args[1])); -} - - -/************************************************************* - * String manipulation - *************************************************************/ - - -/* Convert the argument to a string. Paths are *not* copied to the - store, so `toString /foo/bar' yields `"/foo/bar"', not - `"/nix/store/whatever..."'. */ -static void prim_toString(EvalState & state, Value * * args, Value & v) -{ - PathSet context; - string s = state.coerceToString(*args[0], context, true, false); - mkString(v, s, context); -} - - -/* `substring start len str' returns the substring of `str' starting - at character position `min(start, stringLength str)' inclusive and - ending at `min(start + len, stringLength str)'. `start' must be - non-negative. */ -static void prim_substring(EvalState & state, Value * * args, Value & v) -{ - int start = state.forceInt(*args[0]); - int len = state.forceInt(*args[1]); - PathSet context; - string s = state.coerceToString(*args[2], context); - - if (start < 0) throw EvalError("negative start position in `substring'"); - - mkString(v, start >= s.size() ? "" : string(s, start, len), context); -} - - -static void prim_stringLength(EvalState & state, Value * * args, Value & v) -{ - PathSet context; - string s = state.coerceToString(*args[0], context); - mkInt(v, s.size()); -} - - -static void prim_unsafeDiscardStringContext(EvalState & state, Value * * args, Value & v) -{ - PathSet context; - string s = state.coerceToString(*args[0], context); - mkString(v, s, PathSet()); -} - - -/* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a - builder without causing the derivation to be built (for instance, - in the derivation that builds NARs in nix-push, when doing - source-only deployment). This primop marks the string context so - that builtins.derivation adds the path to drv.inputSrcs rather than - drv.inputDrvs. */ -static void prim_unsafeDiscardOutputDependency(EvalState & state, Value * * args, Value & v) -{ - PathSet context; - string s = state.coerceToString(*args[0], context); - - PathSet context2; - foreach (PathSet::iterator, i, context) { - Path p = *i; - if (p.at(0) == '=') p = "~" + string(p, 1); - context2.insert(p); - } - - mkString(v, s, context2); -} - - -/* Return the cryptographic hash of a string in base-16. */ -static void prim_hashString(EvalState & state, Value * * args, Value & v) -{ - string type = state.forceStringNoCtx(*args[0]); - HashType ht = parseHashType(type); - if (ht == htUnknown) - throw Error(format("unknown hash type `%1%'") % type); - - PathSet context; // discarded - string s = state.forceString(*args[1], context); - - mkString(v, printHash(hashString(ht, s)), context); -}; - - -/************************************************************* - * Versions - *************************************************************/ - - -static void prim_parseDrvName(EvalState & state, Value * * args, Value & v) -{ - string name = state.forceStringNoCtx(*args[0]); - DrvName parsed(name); - state.mkAttrs(v, 2); - mkString(*state.allocAttr(v, state.sName), parsed.name); - mkString(*state.allocAttr(v, state.symbols.create("version")), parsed.version); - v.attrs->sort(); -} - - -static void prim_compareVersions(EvalState & state, Value * * args, Value & v) -{ - string version1 = state.forceStringNoCtx(*args[0]); - string version2 = state.forceStringNoCtx(*args[1]); - mkInt(v, compareVersions(version1, version2)); -} - - -/************************************************************* - * Primop registration - *************************************************************/ - - -void EvalState::createBaseEnv() -{ - baseEnv.up = 0; - - /* Add global constants such as `true' to the base environment. */ - Value v; - - /* `builtins' must be first! */ - mkAttrs(v, 128); - addConstant("builtins", v); - - mkBool(v, true); - addConstant("true", v); - - mkBool(v, false); - addConstant("false", v); - - mkNull(v); - addConstant("null", v); - - mkInt(v, time(0)); - addConstant("__currentTime", v); - - mkString(v, settings.thisSystem.c_str()); - addConstant("__currentSystem", v); - - mkString(v, nixVersion.c_str()); - addConstant("__nixVersion", v); - - /* Language version. This should be increased every time a new - language feature gets added. It's not necessary to increase it - when primops get added, because you can just use `builtins ? - primOp' to check. */ - mkInt(v, 2); - addConstant("__langVersion", v); - - // Miscellaneous - addPrimOp("import", 1, prim_import); - addPrimOp("__typeOf", 1, prim_typeOf); - addPrimOp("isNull", 1, prim_isNull); - addPrimOp("__isFunction", 1, prim_isFunction); - addPrimOp("__isString", 1, prim_isString); - addPrimOp("__isInt", 1, prim_isInt); - addPrimOp("__isBool", 1, prim_isBool); - addPrimOp("__genericClosure", 1, prim_genericClosure); - addPrimOp("abort", 1, prim_abort); - addPrimOp("throw", 1, prim_throw); - addPrimOp("__addErrorContext", 2, prim_addErrorContext); - addPrimOp("__tryEval", 1, prim_tryEval); - addPrimOp("__getEnv", 1, prim_getEnv); - addPrimOp("__trace", 2, prim_trace); - - // Paths - addPrimOp("__toPath", 1, prim_toPath); - addPrimOp("__storePath", 1, prim_storePath); - addPrimOp("__pathExists", 1, prim_pathExists); - addPrimOp("baseNameOf", 1, prim_baseNameOf); - addPrimOp("dirOf", 1, prim_dirOf); - addPrimOp("__readFile", 1, prim_readFile); - - // Creating files - addPrimOp("__toXML", 1, prim_toXML); - addPrimOp("__toJSON", 1, prim_toJSON); - addPrimOp("__toFile", 2, prim_toFile); - addPrimOp("__filterSource", 2, prim_filterSource); - - // Sets - addPrimOp("__attrNames", 1, prim_attrNames); - addPrimOp("__getAttr", 2, prim_getAttr); - addPrimOp("__unsafeGetAttrPos", 2, prim_unsafeGetAttrPos); - addPrimOp("__hasAttr", 2, prim_hasAttr); - addPrimOp("__isAttrs", 1, prim_isAttrs); - addPrimOp("removeAttrs", 2, prim_removeAttrs); - addPrimOp("__listToAttrs", 1, prim_listToAttrs); - addPrimOp("__intersectAttrs", 2, prim_intersectAttrs); - addPrimOp("__functionArgs", 1, prim_functionArgs); - - // Lists - addPrimOp("__isList", 1, prim_isList); - addPrimOp("__elemAt", 2, prim_elemAt); - addPrimOp("__head", 1, prim_head); - addPrimOp("__tail", 1, prim_tail); - addPrimOp("map", 2, prim_map); - addPrimOp("__filter", 2, prim_filter); - addPrimOp("__elem", 2, prim_elem); - addPrimOp("__concatLists", 1, prim_concatLists); - addPrimOp("__length", 1, prim_length); - - // Integer arithmetic - addPrimOp("__add", 2, prim_add); - addPrimOp("__sub", 2, prim_sub); - addPrimOp("__mul", 2, prim_mul); - addPrimOp("__div", 2, prim_div); - addPrimOp("__lessThan", 2, prim_lessThan); - - // String manipulation - addPrimOp("toString", 1, prim_toString); - addPrimOp("__substring", 3, prim_substring); - addPrimOp("__stringLength", 1, prim_stringLength); - addPrimOp("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext); - addPrimOp("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency); - addPrimOp("__hashString", 2, prim_hashString); - - // Versions - addPrimOp("__parseDrvName", 1, prim_parseDrvName); - addPrimOp("__compareVersions", 2, prim_compareVersions); - - // Derivations - addPrimOp("derivationStrict", 1, prim_derivationStrict); - - /* Add a wrapper around the derivation primop that computes the - `drvPath' and `outPath' attributes lazily. */ - string path = findFile("nix/derivation.nix"); - assert(!path.empty()); - sDerivationNix = symbols.create(path); - evalFile(path, v); - addConstant("derivation", v); - - /* Now that we've added all primops, sort the `builtins' set, - because attribute lookups expect it to be sorted. */ - baseEnv.values[0]->attrs->sort(); -} - - -} diff --git a/src/libexpr/symbol-table.hh b/src/libexpr/symbol-table.hh deleted file mode 100644 index 08e31d4965..0000000000 --- a/src/libexpr/symbol-table.hh +++ /dev/null @@ -1,95 +0,0 @@ -#pragma once - -#include "config.h" - -#include <map> - -#if HAVE_TR1_UNORDERED_SET -#include <tr1/unordered_set> -#endif - -#include "types.hh" - -namespace nix { - -/* Symbol table used by the parser and evaluator to represent and look - up identifiers and attributes efficiently. SymbolTable::create() - converts a string into a symbol. Symbols have the property that - they can be compared efficiently (using a pointer equality test), - because the symbol table stores only one copy of each string. */ - -class Symbol -{ -private: - const string * s; // pointer into SymbolTable - Symbol(const string * s) : s(s) { }; - friend class SymbolTable; - -public: - Symbol() : s(0) { }; - - bool operator == (const Symbol & s2) const - { - return s == s2.s; - } - - bool operator != (const Symbol & s2) const - { - return s != s2.s; - } - - bool operator < (const Symbol & s2) const - { - return s < s2.s; - } - - operator const string & () const - { - return *s; - } - - bool set() const - { - return s; - } - - bool empty() const - { - return s->empty(); - } - - friend std::ostream & operator << (std::ostream & str, const Symbol & sym); -}; - -inline std::ostream & operator << (std::ostream & str, const Symbol & sym) -{ - str << *sym.s; - return str; -} - -class SymbolTable -{ -private: -#if HAVE_TR1_UNORDERED_SET - typedef std::tr1::unordered_set<string> Symbols; -#else - typedef std::set<string> Symbols; -#endif - Symbols symbols; - -public: - Symbol create(const string & s) - { - std::pair<Symbols::iterator, bool> res = symbols.insert(s); - return Symbol(&*res.first); - } - - unsigned int size() const - { - return symbols.size(); - } - - size_t totalSize() const; -}; - -} diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc deleted file mode 100644 index a2004df5c8..0000000000 --- a/src/libexpr/value-to-json.cc +++ /dev/null @@ -1,86 +0,0 @@ -#include "value-to-json.hh" -#include "eval-inline.hh" -#include "util.hh" - -#include <cstdlib> - - -namespace nix { - - -void escapeJSON(std::ostream & str, const string & s) -{ - str << "\""; - foreach (string::const_iterator, i, s) - if (*i == '\"' || *i == '\\') str << "\\" << *i; - else if (*i == '\n') str << "\\n"; - else if (*i == '\r') str << "\\r"; - else if (*i == '\t') str << "\\t"; - else str << *i; - str << "\""; -} - - -void printValueAsJSON(EvalState & state, bool strict, - Value & v, std::ostream & str, PathSet & context) -{ - checkInterrupt(); - - if (strict) state.forceValue(v); - - switch (v.type) { - - case tInt: - str << v.integer; - break; - - case tBool: - str << (v.boolean ? "true" : "false"); - break; - - case tString: - copyContext(v, context); - escapeJSON(str, v.string.s); - break; - - case tPath: - escapeJSON(str, state.copyPathToStore(context, v.path)); - break; - - case tNull: - str << "null"; - break; - - case tAttrs: { - Bindings::iterator i = v.attrs->find(state.sOutPath); - if (i == v.attrs->end()) { - JSONObject json(str); - StringSet names; - foreach (Bindings::iterator, i, *v.attrs) - names.insert(i->name); - foreach (StringSet::iterator, i, names) { - Attr & a(*v.attrs->find(state.symbols.create(*i))); - json.attr(*i); - printValueAsJSON(state, strict, *a.value, str, context); - } - } else - printValueAsJSON(state, strict, *i->value, str, context); - break; - } - - case tList: { - JSONList json(str); - for (unsigned int n = 0; n < v.list.length; ++n) { - json.elem(); - printValueAsJSON(state, strict, *v.list.elems[n], str, context); - } - break; - } - - default: - throw TypeError(format("cannot convert %1% to JSON") % showType(v)); - } -} - - -} diff --git a/src/libexpr/value-to-json.hh b/src/libexpr/value-to-json.hh deleted file mode 100644 index e3a97efe42..0000000000 --- a/src/libexpr/value-to-json.hh +++ /dev/null @@ -1,64 +0,0 @@ -#pragma once - -#include "nixexpr.hh" -#include "eval.hh" - -#include <string> -#include <map> - -namespace nix { - -void printValueAsJSON(EvalState & state, bool strict, - Value & v, std::ostream & out, PathSet & context); - -void escapeJSON(std::ostream & str, const string & s); - -struct JSONObject -{ - std::ostream & str; - bool first; - JSONObject(std::ostream & str) : str(str), first(true) - { - str << "{"; - } - ~JSONObject() - { - str << "}"; - } - void attr(const string & s) - { - if (!first) str << ","; else first = false; - escapeJSON(str, s); - str << ":"; - } - void attr(const string & s, const string & t) - { - attr(s); - escapeJSON(str, t); - } -}; - -struct JSONList -{ - std::ostream & str; - bool first; - JSONList(std::ostream & str) : str(str), first(true) - { - str << "["; - } - ~JSONList() - { - str << "]"; - } - void elem() - { - if (!first) str << ","; else first = false; - } - void elem(const string & s) - { - elem(); - escapeJSON(str, s); - } -}; - -} diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc deleted file mode 100644 index 3934a83eec..0000000000 --- a/src/libexpr/value-to-xml.cc +++ /dev/null @@ -1,163 +0,0 @@ -#include "value-to-xml.hh" -#include "xml-writer.hh" -#include "eval-inline.hh" -#include "util.hh" - -#include <cstdlib> - - -namespace nix { - - -static XMLAttrs singletonAttrs(const string & name, const string & value) -{ - XMLAttrs attrs; - attrs[name] = value; - return attrs; -} - - -static void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen); - - -static void posToXML(XMLAttrs & xmlAttrs, const Pos & pos) -{ - xmlAttrs["path"] = pos.file; - xmlAttrs["line"] = (format("%1%") % pos.line).str(); - xmlAttrs["column"] = (format("%1%") % pos.column).str(); -} - - -static void showAttrs(EvalState & state, bool strict, bool location, - Bindings & attrs, XMLWriter & doc, PathSet & context, PathSet & drvsSeen) -{ - StringSet names; - - foreach (Bindings::iterator, i, attrs) - names.insert(i->name); - - foreach (StringSet::iterator, i, names) { - Attr & a(*attrs.find(state.symbols.create(*i))); - - XMLAttrs xmlAttrs; - xmlAttrs["name"] = *i; - if (location && a.pos != &noPos) posToXML(xmlAttrs, *a.pos); - - XMLOpenElement _(doc, "attr", xmlAttrs); - printValueAsXML(state, strict, location, - *a.value, doc, context, drvsSeen); - } -} - - -static void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen) -{ - checkInterrupt(); - - if (strict) state.forceValue(v); - - switch (v.type) { - - case tInt: - doc.writeEmptyElement("int", singletonAttrs("value", (format("%1%") % v.integer).str())); - break; - - case tBool: - doc.writeEmptyElement("bool", singletonAttrs("value", v.boolean ? "true" : "false")); - break; - - case tString: - /* !!! show the context? */ - copyContext(v, context); - doc.writeEmptyElement("string", singletonAttrs("value", v.string.s)); - break; - - case tPath: - doc.writeEmptyElement("path", singletonAttrs("value", v.path)); - break; - - case tNull: - doc.writeEmptyElement("null"); - break; - - case tAttrs: - if (state.isDerivation(v)) { - XMLAttrs xmlAttrs; - - Bindings::iterator a = v.attrs->find(state.symbols.create("derivation")); - - Path drvPath; - a = v.attrs->find(state.sDrvPath); - if (a != v.attrs->end()) { - if (strict) state.forceValue(*a->value); - if (a->value->type == tString) - xmlAttrs["drvPath"] = drvPath = a->value->string.s; - } - - a = v.attrs->find(state.sOutPath); - if (a != v.attrs->end()) { - if (strict) state.forceValue(*a->value); - if (a->value->type == tString) - xmlAttrs["outPath"] = a->value->string.s; - } - - XMLOpenElement _(doc, "derivation", xmlAttrs); - - if (drvPath != "" && drvsSeen.find(drvPath) == drvsSeen.end()) { - drvsSeen.insert(drvPath); - showAttrs(state, strict, location, *v.attrs, doc, context, drvsSeen); - } else - doc.writeEmptyElement("repeated"); - } - - else { - XMLOpenElement _(doc, "attrs"); - showAttrs(state, strict, location, *v.attrs, doc, context, drvsSeen); - } - - break; - - case tList: { - XMLOpenElement _(doc, "list"); - for (unsigned int n = 0; n < v.list.length; ++n) - printValueAsXML(state, strict, location, *v.list.elems[n], doc, context, drvsSeen); - break; - } - - case tLambda: { - XMLAttrs xmlAttrs; - if (location) posToXML(xmlAttrs, v.lambda.fun->pos); - XMLOpenElement _(doc, "function", xmlAttrs); - - if (v.lambda.fun->matchAttrs) { - XMLAttrs attrs; - if (!v.lambda.fun->arg.empty()) attrs["name"] = v.lambda.fun->arg; - if (v.lambda.fun->formals->ellipsis) attrs["ellipsis"] = "1"; - XMLOpenElement _(doc, "attrspat", attrs); - foreach (Formals::Formals_::iterator, i, v.lambda.fun->formals->formals) - doc.writeEmptyElement("attr", singletonAttrs("name", i->name)); - } else - doc.writeEmptyElement("varpat", singletonAttrs("name", v.lambda.fun->arg)); - - break; - } - - default: - doc.writeEmptyElement("unevaluated"); - } -} - - -void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, std::ostream & out, PathSet & context) -{ - XMLWriter doc(true, out); - XMLOpenElement root(doc, "expr"); - PathSet drvsSeen; - printValueAsXML(state, strict, location, v, doc, context, drvsSeen); -} - - -} diff --git a/src/libexpr/value-to-xml.hh b/src/libexpr/value-to-xml.hh deleted file mode 100644 index 97657327ed..0000000000 --- a/src/libexpr/value-to-xml.hh +++ /dev/null @@ -1,14 +0,0 @@ -#pragma once - -#include "nixexpr.hh" -#include "eval.hh" - -#include <string> -#include <map> - -namespace nix { - -void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, std::ostream & out, PathSet & context); - -} diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh deleted file mode 100644 index 2feb2f9492..0000000000 --- a/src/libexpr/value.hh +++ /dev/null @@ -1,162 +0,0 @@ -#pragma once - -#include "symbol-table.hh" - -namespace nix { - - -typedef enum { - tInt = 1, - tBool, - tString, - tPath, - tNull, - tAttrs, - tList, - tThunk, - tApp, - tLambda, - tBlackhole, - tPrimOp, - tPrimOpApp, -} ValueType; - - -class Bindings; -struct Env; -struct Expr; -struct ExprLambda; -struct PrimOp; -struct PrimOp; -class Symbol; - - -typedef long NixInt; - - -struct Value -{ - ValueType type; - union - { - NixInt integer; - bool boolean; - - /* Strings in the evaluator carry a so-called `context' which - is a list of strings representing store paths. This is to - allow users to write things like - - "--with-freetype2-library=" + freetype + "/lib" - - where `freetype' is a derivation (or a source to be copied - to the store). If we just concatenated the strings without - keeping track of the referenced store paths, then if the - string is used as a derivation attribute, the derivation - will not have the correct dependencies in its inputDrvs and - inputSrcs. - - The semantics of the context is as follows: when a string - with context C is used as a derivation attribute, then the - derivations in C will be added to the inputDrvs of the - derivation, and the other store paths in C will be added to - the inputSrcs of the derivations. - - For canonicity, the store paths should be in sorted order. */ - struct { - const char * s; - const char * * context; // must be in sorted order - } string; - - const char * path; - Bindings * attrs; - struct { - unsigned int length; - Value * * elems; - } list; - struct { - Env * env; - Expr * expr; - } thunk; - struct { - Value * left, * right; - } app; - struct { - Env * env; - ExprLambda * fun; - } lambda; - PrimOp * primOp; - struct { - Value * left, * right; - } primOpApp; - }; -}; - - -/* After overwriting an app node, be sure to clear pointers in the - Value to ensure that the target isn't kept alive unnecessarily. */ -static inline void clearValue(Value & v) -{ - v.app.right = 0; -} - - -static inline void mkInt(Value & v, NixInt n) -{ - clearValue(v); - v.type = tInt; - v.integer = n; -} - - -static inline void mkBool(Value & v, bool b) -{ - clearValue(v); - v.type = tBool; - v.boolean = b; -} - - -static inline void mkNull(Value & v) -{ - v.type = tNull; - v.app.left = v.app.right = 00; // scrub -} - - -static inline void mkApp(Value & v, Value & left, Value & right) -{ - v.type = tApp; - v.app.left = &left; - v.app.right = &right; -} - - -static inline void mkStringNoCopy(Value & v, const char * s) -{ - v.type = tString; - v.string.s = s; - v.string.context = 0; -} - - -static inline void mkString(Value & v, const Symbol & s) -{ - mkStringNoCopy(v, ((const string &) s).c_str()); -} - - -void mkString(Value & v, const char * s); - - -static inline void mkPathNoCopy(Value & v, const char * s) -{ - clearValue(v); - v.type = tPath; - v.path = s; -} - - -void mkPath(Value & v, const char * s); - - -} diff --git a/src/libmain/local.mk b/src/libmain/local.mk deleted file mode 100644 index 71a07d1979..0000000000 --- a/src/libmain/local.mk +++ /dev/null @@ -1,11 +0,0 @@ -libraries += libmain - -libmain_NAME = libnixmain - -libmain_DIR := $(d) - -libmain_SOURCES := $(wildcard $(d)/*.cc) - -libmain_LIBS = libstore libutil libformat - -libmain_ALLOW_UNDEFINED = 1 diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc deleted file mode 100644 index a390654452..0000000000 --- a/src/libmain/shared.cc +++ /dev/null @@ -1,300 +0,0 @@ -#include "config.h" - -#include "shared.hh" -#include "globals.hh" -#include "store-api.hh" -#include "util.hh" -#include "misc.hh" - -#include <iostream> -#include <cctype> -#include <exception> - -#include <sys/time.h> -#include <sys/stat.h> -#include <unistd.h> -#include <signal.h> - - -namespace nix { - - -volatile sig_atomic_t blockInt = 0; - - -static void sigintHandler(int signo) -{ - if (!blockInt) { - _isInterrupted = 1; - blockInt = 1; - } -} - - -static bool gcWarning = true; - -void printGCWarning() -{ - if (!gcWarning) return; - static bool haveWarned = false; - warnOnce(haveWarned, - "you did not specify `--add-root'; " - "the result might be removed by the garbage collector"); -} - - -void printMissing(StoreAPI & store, const PathSet & paths) -{ - unsigned long long downloadSize, narSize; - PathSet willBuild, willSubstitute, unknown; - queryMissing(store, paths, willBuild, willSubstitute, unknown, downloadSize, narSize); - printMissing(willBuild, willSubstitute, unknown, downloadSize, narSize); -} - - -void printMissing(const PathSet & willBuild, - const PathSet & willSubstitute, const PathSet & unknown, - unsigned long long downloadSize, unsigned long long narSize) -{ - if (!willBuild.empty()) { - printMsg(lvlInfo, format("these derivations will be built:")); - foreach (PathSet::iterator, i, willBuild) - printMsg(lvlInfo, format(" %1%") % *i); - } - - if (!willSubstitute.empty()) { - printMsg(lvlInfo, format("these paths will be fetched (%.2f MiB download, %.2f MiB unpacked):") - % (downloadSize / (1024.0 * 1024.0)) - % (narSize / (1024.0 * 1024.0))); - foreach (PathSet::iterator, i, willSubstitute) - printMsg(lvlInfo, format(" %1%") % *i); - } - - if (!unknown.empty()) { - printMsg(lvlInfo, format("don't know how to build these paths%1%:") - % (settings.readOnlyMode ? " (may be caused by read-only store access)" : "")); - foreach (PathSet::iterator, i, unknown) - printMsg(lvlInfo, format(" %1%") % *i); - } -} - - -static void setLogType(string lt) -{ - if (lt == "pretty") logType = ltPretty; - else if (lt == "escapes") logType = ltEscapes; - else if (lt == "flat") logType = ltFlat; - else throw UsageError("unknown log type"); -} - - -string getArg(const string & opt, - Strings::iterator & i, const Strings::iterator & end) -{ - ++i; - if (i == end) throw UsageError(format("`%1%' requires an argument") % opt); - return *i; -} - - -void detectStackOverflow(); - - -/* Initialize and reorder arguments, then call the actual argument - processor. */ -static void initAndRun(int argc, char * * argv) -{ - settings.processEnvironment(); - settings.loadConfFile(); - - /* Catch SIGINT. */ - struct sigaction act; - act.sa_handler = sigintHandler; - sigemptyset(&act.sa_mask); - act.sa_flags = 0; - if (sigaction(SIGINT, &act, 0)) - throw SysError("installing handler for SIGINT"); - if (sigaction(SIGTERM, &act, 0)) - throw SysError("installing handler for SIGTERM"); - if (sigaction(SIGHUP, &act, 0)) - throw SysError("installing handler for SIGHUP"); - - /* Ignore SIGPIPE. */ - act.sa_handler = SIG_IGN; - act.sa_flags = 0; - if (sigaction(SIGPIPE, &act, 0)) - throw SysError("ignoring SIGPIPE"); - - /* Reset SIGCHLD to its default. */ - act.sa_handler = SIG_DFL; - act.sa_flags = 0; - if (sigaction(SIGCHLD, &act, 0)) - throw SysError("resetting SIGCHLD"); - - /* Register a SIGSEGV handler to detect stack overflows. */ - detectStackOverflow(); - - /* There is no privacy in the Nix system ;-) At least not for - now. In particular, store objects should be readable by - everybody. */ - umask(0022); - - /* Initialise the PRNG. */ - struct timeval tv; - gettimeofday(&tv, 0); - srandom(tv.tv_usec); - - /* Process the NIX_LOG_TYPE environment variable. */ - string lt = getEnv("NIX_LOG_TYPE"); - if (lt != "") setLogType(lt); - - /* Put the arguments in a vector. */ - Strings args, remaining; - while (argc--) args.push_back(*argv++); - args.erase(args.begin()); - - /* Expand compound dash options (i.e., `-qlf' -> `-q -l -f'), and - ignore options for the ATerm library. */ - for (Strings::iterator i = args.begin(); i != args.end(); ++i) { - string arg = *i; - if (arg.length() > 2 && arg[0] == '-' && arg[1] != '-' && !isdigit(arg[1])) { - for (unsigned int j = 1; j < arg.length(); j++) - if (isalpha(arg[j])) - remaining.push_back((string) "-" + arg[j]); - else { - remaining.push_back(string(arg, j)); - break; - } - } else remaining.push_back(arg); - } - args = remaining; - remaining.clear(); - - /* Process default options. */ - for (Strings::iterator i = args.begin(); i != args.end(); ++i) { - string arg = *i; - if (arg == "--verbose" || arg == "-v") verbosity = (Verbosity) (verbosity + 1); - else if (arg == "--quiet") verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; - else if (arg == "--log-type") { - string s = getArg(arg, i, args.end()); - setLogType(s); - } - else if (arg == "--no-build-output" || arg == "-Q") - settings.buildVerbosity = lvlVomit; - else if (arg == "--print-build-trace") - settings.printBuildTrace = true; - else if (arg == "--help") { - printHelp(); - return; - } - else if (arg == "--version") { - std::cout << format("%1% (Nix) %2%") % programId % nixVersion << std::endl; - return; - } - else if (arg == "--keep-failed" || arg == "-K") - settings.keepFailed = true; - else if (arg == "--keep-going" || arg == "-k") - settings.keepGoing = true; - else if (arg == "--fallback") - settings.set("build-fallback", "true"); - else if (arg == "--max-jobs" || arg == "-j") - settings.set("build-max-jobs", getArg(arg, i, args.end())); - else if (arg == "--cores") - settings.set("build-cores", getArg(arg, i, args.end())); - else if (arg == "--readonly-mode") - settings.readOnlyMode = true; - else if (arg == "--max-silent-time") - settings.set("build-max-silent-time", getArg(arg, i, args.end())); - else if (arg == "--timeout") - settings.set("build-timeout", getArg(arg, i, args.end())); - else if (arg == "--no-build-hook") - settings.useBuildHook = false; - else if (arg == "--show-trace") - settings.showTrace = true; - else if (arg == "--no-gc-warning") - gcWarning = false; - else if (arg == "--option") { - ++i; if (i == args.end()) throw UsageError("`--option' requires two arguments"); - string name = *i; - ++i; if (i == args.end()) throw UsageError("`--option' requires two arguments"); - string value = *i; - settings.set(name, value); - } - else remaining.push_back(arg); - } - - if (char *pack = getenv("_NIX_OPTIONS")) - settings.unpack(pack); - - settings.update(); - - run(remaining); - - /* Close the Nix database. */ - store.reset((StoreAPI *) 0); -} - - -void showManPage(const string & name) -{ - string cmd = "man " + name; - if (system(cmd.c_str()) != 0) - throw Error(format("command `%1%' failed") % cmd); -} - - -int exitCode = 0; -char * * argvSaved = 0; - -} - - -static char buf[1024]; - -int main(int argc, char * * argv) -{ - using namespace nix; - - argvSaved = argv; - - /* Turn on buffering for cerr. */ -#if HAVE_PUBSETBUF - std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf)); -#endif - - std::ios::sync_with_stdio(false); - - try { - try { - initAndRun(argc, argv); - } catch (...) { - /* Subtle: we have to make sure that any `interrupted' - condition is discharged before we reach printMsg() - below, since otherwise it will throw an (uncaught) - exception. */ - blockInt = 1; /* ignore further SIGINTs */ - _isInterrupted = 0; - throw; - } - } catch (UsageError & e) { - printMsg(lvlError, - format( - "error: %1%\n" - "Try `%2% --help' for more information.") - % e.what() % programId); - return 1; - } catch (BaseError & e) { - printMsg(lvlError, format("error: %1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg()); - if (e.prefix() != "" && !settings.showTrace) - printMsg(lvlError, "(use `--show-trace' to show detailed location information)"); - return e.status; - } catch (std::bad_alloc & e) { - printMsg(lvlError, "error: out of memory"); - return 1; - } catch (std::exception & e) { - printMsg(lvlError, format("error: %1%") % e.what()); - return 1; - } - - return exitCode; -} diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh deleted file mode 100644 index b29b08bb59..0000000000 --- a/src/libmain/shared.hh +++ /dev/null @@ -1,73 +0,0 @@ -#pragma once - -#include "util.hh" - -#include <signal.h> - -#include <locale> - - -/* These are not implemented here, but must be implemented by a - program linking against libmain. */ - -/* Main program. Called by main() after the ATerm library has been - initialised and some default arguments have been processed (and - removed from `args'). main() will catch all exceptions. */ -void run(nix::Strings args); - -/* Should print a help message to stdout and return. */ -void printHelp(); - -extern std::string programId; - - -namespace nix { - -MakeError(UsageError, nix::Error); - -class StoreAPI; - -/* Ugh. No better place to put this. */ -void printGCWarning(); - -void printMissing(StoreAPI & store, const PathSet & paths); - -void printMissing(const PathSet & willBuild, - const PathSet & willSubstitute, const PathSet & unknown, - unsigned long long downloadSize, unsigned long long narSize); - -template<class N> N getIntArg(const string & opt, - Strings::iterator & i, const Strings::iterator & end, bool allowUnit) -{ - ++i; - if (i == end) throw UsageError(format("`%1%' requires an argument") % opt); - string s = *i; - N multiplier = 1; - if (allowUnit && !s.empty()) { - char u = std::toupper(*s.rbegin()); - if (std::isalpha(u)) { - if (u == 'K') multiplier = 1ULL << 10; - else if (u == 'M') multiplier = 1ULL << 20; - else if (u == 'G') multiplier = 1ULL << 30; - else if (u == 'T') multiplier = 1ULL << 40; - else throw UsageError(format("invalid unit specifier `%1%'") % u); - s.resize(s.size() - 1); - } - } - N n; - if (!string2Int(s, n)) - throw UsageError(format("`%1%' requires an integer argument") % opt); - return n * multiplier; -} - -/* Show the manual page for the specified program. */ -void showManPage(const string & name); - -extern volatile ::sig_atomic_t blockInt; - -/* Exit code of the program. */ -extern int exitCode; - -extern char * * argvSaved; - -} diff --git a/src/libmain/stack.cc b/src/libmain/stack.cc deleted file mode 100644 index 64df95547e..0000000000 --- a/src/libmain/stack.cc +++ /dev/null @@ -1,72 +0,0 @@ -#include "config.h" - -#include "types.hh" - -#include <cstring> -#include <cstddef> -#include <cstdlib> - -#include <unistd.h> -#include <signal.h> - -namespace nix { - - -static void sigsegvHandler(int signo, siginfo_t * info, void * ctx) -{ - /* Detect stack overflows by comparing the faulting address with - the stack pointer. Unfortunately, getting the stack pointer is - not portable. */ - bool haveSP = true; - char * sp; -#if defined(__x86_64__) && defined(REG_RSP) - sp = (char *) ((ucontext *) ctx)->uc_mcontext.gregs[REG_RSP]; -#elif defined(REG_ESP) - sp = (char *) ((ucontext *) ctx)->uc_mcontext.gregs[REG_ESP]; -#else - haveSP = false; -#endif - - if (haveSP) { - ptrdiff_t diff = (char *) info->si_addr - sp; - if (diff < 0) diff = -diff; - if (diff < 4096) { - char msg[] = "error: stack overflow (possible infinite recursion)\n"; - write(2, msg, strlen(msg)); - _exit(1); // maybe abort instead? - } - } - - /* Restore default behaviour (i.e. segfault and dump core). */ - struct sigaction act; - sigfillset(&act.sa_mask); - act.sa_handler = SIG_DFL; - act.sa_flags = 0; - if (sigaction(SIGSEGV, &act, 0)) abort(); -} - - -void detectStackOverflow() -{ -#if defined(SA_SIGINFO) && defined (SA_ONSTACK) - /* Install a SIGSEGV handler to detect stack overflows. This - requires an alternative stack, otherwise the signal cannot be - delivered when we're out of stack space. */ - stack_t stack; - stack.ss_size = 4096 * 4 + MINSIGSTKSZ; - stack.ss_sp = new char[stack.ss_size]; - if (!stack.ss_sp) throw Error("cannot allocate alternative stack"); - stack.ss_flags = 0; - if (sigaltstack(&stack, 0) == -1) throw SysError("cannot set alternative stack"); - - struct sigaction act; - sigfillset(&act.sa_mask); - act.sa_sigaction = sigsegvHandler; - act.sa_flags = SA_SIGINFO | SA_ONSTACK; - if (sigaction(SIGSEGV, &act, 0)) - throw SysError("resetting SIGCHLD"); -#endif -} - - -} diff --git a/src/libstore/build.cc b/src/libstore/build.cc deleted file mode 100644 index 2e2f92fadf..0000000000 --- a/src/libstore/build.cc +++ /dev/null @@ -1,3351 +0,0 @@ -#include "config.h" - -#include "references.hh" -#include "pathlocks.hh" -#include "misc.hh" -#include "globals.hh" -#include "local-store.hh" -#include "util.hh" -#include "archive.hh" -#include "affinity.hh" - -#include <map> -#include <sstream> -#include <algorithm> - -#include <time.h> -#include <sys/time.h> -#include <sys/wait.h> -#include <sys/types.h> -#include <sys/stat.h> -#include <sys/utsname.h> -#include <fcntl.h> -#include <unistd.h> -#include <errno.h> -#include <stdio.h> -#include <cstring> - -#include <pwd.h> -#include <grp.h> - -#include <bzlib.h> - -/* Includes required for chroot support. */ -#if HAVE_SYS_PARAM_H -#include <sys/param.h> -#endif -#if HAVE_SYS_MOUNT_H -#include <sys/mount.h> -#endif -#if HAVE_SCHED_H -#include <sched.h> -#endif - -/* In GNU libc 2.11, <sys/mount.h> does not define `MS_PRIVATE', but - <linux/fs.h> does. */ -#if !defined MS_PRIVATE && defined HAVE_LINUX_FS_H -#include <linux/fs.h> -#endif - -#define CHROOT_ENABLED HAVE_CHROOT && HAVE_UNSHARE && HAVE_SYS_MOUNT_H && defined(MS_BIND) && defined(MS_PRIVATE) && defined(CLONE_NEWNS) - -#if CHROOT_ENABLED -#include <sys/socket.h> -#include <sys/ioctl.h> -#include <net/if.h> -#include <netinet/ip.h> -#endif - -#if HAVE_SYS_PERSONALITY_H -#include <sys/personality.h> -#define CAN_DO_LINUX32_BUILDS -#endif - -#if HAVE_STATVFS -#include <sys/statvfs.h> -#endif - - -namespace nix { - -using std::map; - - -static string pathNullDevice = "/dev/null"; - - -/* Forward definition. */ -class Worker; -struct HookInstance; - - -/* A pointer to a goal. */ -class Goal; -typedef std::shared_ptr<Goal> GoalPtr; -typedef std::weak_ptr<Goal> WeakGoalPtr; - -/* Set of goals. */ -typedef set<GoalPtr> Goals; -typedef list<WeakGoalPtr> WeakGoals; - -/* A map of paths to goals (and the other way around). */ -typedef map<Path, WeakGoalPtr> WeakGoalMap; - - - -class Goal : public std::enable_shared_from_this<Goal> -{ -public: - typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode; - -protected: - - /* Backlink to the worker. */ - Worker & worker; - - /* Goals that this goal is waiting for. */ - Goals waitees; - - /* Goals waiting for this one to finish. Must use weak pointers - here to prevent cycles. */ - WeakGoals waiters; - - /* Number of goals we are/were waiting for that have failed. */ - unsigned int nrFailed; - - /* Number of substitution goals we are/were waiting for that - failed because there are no substituters. */ - unsigned int nrNoSubstituters; - - /* Number of substitution goals we are/were waiting for that - failed because othey had unsubstitutable references. */ - unsigned int nrIncompleteClosure; - - /* Name of this goal for debugging purposes. */ - string name; - - /* Whether the goal is finished. */ - ExitCode exitCode; - - Goal(Worker & worker) : worker(worker) - { - nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; - exitCode = ecBusy; - } - - virtual ~Goal() - { - trace("goal destroyed"); - } - -public: - virtual void work() = 0; - - void addWaitee(GoalPtr waitee); - - virtual void waiteeDone(GoalPtr waitee, ExitCode result); - - virtual void handleChildOutput(int fd, const string & data) - { - abort(); - } - - virtual void handleEOF(int fd) - { - abort(); - } - - void trace(const format & f); - - string getName() - { - return name; - } - - ExitCode getExitCode() - { - return exitCode; - } - - /* Cancel the goal. It should wake up its waiters, get rid of any - running child processes that are being monitored by the worker - (important!), etc. */ - virtual void cancel(bool timeout) = 0; - -protected: - void amDone(ExitCode result); -}; - - -/* A mapping used to remember for each child process to what goal it - belongs, and file descriptors for receiving log data and output - path creation commands. */ -struct Child -{ - WeakGoalPtr goal; - set<int> fds; - bool respectTimeouts; - bool inBuildSlot; - time_t lastOutput; /* time we last got output on stdout/stderr */ - time_t timeStarted; -}; - -typedef map<pid_t, Child> Children; - - -/* The worker class. */ -class Worker -{ -private: - - /* Note: the worker should only have strong pointers to the - top-level goals. */ - - /* The top-level goals of the worker. */ - Goals topGoals; - - /* Goals that are ready to do some work. */ - WeakGoals awake; - - /* Goals waiting for a build slot. */ - WeakGoals wantingToBuild; - - /* Child processes currently running. */ - Children children; - - /* Number of build slots occupied. This includes local builds and - substitutions but not remote builds via the build hook. */ - unsigned int nrLocalBuilds; - - /* Maps used to prevent multiple instantiations of a goal for the - same derivation / path. */ - WeakGoalMap derivationGoals; - WeakGoalMap substitutionGoals; - - /* Goals waiting for busy paths to be unlocked. */ - WeakGoals waitingForAnyGoal; - - /* Goals sleeping for a few seconds (polling a lock). */ - WeakGoals waitingForAWhile; - - /* Last time the goals in `waitingForAWhile' where woken up. */ - time_t lastWokenUp; - -public: - - /* Set if at least one derivation had a BuildError (i.e. permanent - failure). */ - bool permanentFailure; - - LocalStore & store; - - std::shared_ptr<HookInstance> hook; - - Worker(LocalStore & store); - ~Worker(); - - /* Make a goal (with caching). */ - GoalPtr makeDerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); - GoalPtr makeSubstitutionGoal(const Path & storePath, bool repair = false); - - /* Remove a dead goal. */ - void removeGoal(GoalPtr goal); - - /* Wake up a goal (i.e., there is something for it to do). */ - void wakeUp(GoalPtr goal); - - /* Return the number of local build and substitution processes - currently running (but not remote builds via the build - hook). */ - unsigned int getNrLocalBuilds(); - - /* Registers a running child process. `inBuildSlot' means that - the process counts towards the jobs limit. */ - void childStarted(GoalPtr goal, pid_t pid, - const set<int> & fds, bool inBuildSlot, bool respectTimeouts); - - /* Unregisters a running child process. `wakeSleepers' should be - false if there is no sense in waking up goals that are sleeping - because they can't run yet (e.g., there is no free build slot, - or the hook would still say `postpone'). */ - void childTerminated(pid_t pid, bool wakeSleepers = true); - - /* Put `goal' to sleep until a build slot becomes available (which - might be right away). */ - void waitForBuildSlot(GoalPtr goal); - - /* Wait for any goal to finish. Pretty indiscriminate way to - wait for some resource that some other goal is holding. */ - void waitForAnyGoal(GoalPtr goal); - - /* Wait for a few seconds and then retry this goal. Used when - waiting for a lock held by another process. This kind of - polling is inefficient, but POSIX doesn't really provide a way - to wait for multiple locks in the main select() loop. */ - void waitForAWhile(GoalPtr goal); - - /* Loop until the specified top-level goals have finished. */ - void run(const Goals & topGoals); - - /* Wait for input to become available. */ - void waitForInput(); - - unsigned int exitStatus(); -}; - - -////////////////////////////////////////////////////////////////////// - - -void addToWeakGoals(WeakGoals & goals, GoalPtr p) -{ - // FIXME: necessary? - foreach (WeakGoals::iterator, i, goals) - if (i->lock() == p) return; - goals.push_back(p); -} - - -void Goal::addWaitee(GoalPtr waitee) -{ - waitees.insert(waitee); - addToWeakGoals(waitee->waiters, shared_from_this()); -} - - -void Goal::waiteeDone(GoalPtr waitee, ExitCode result) -{ - assert(waitees.find(waitee) != waitees.end()); - waitees.erase(waitee); - - trace(format("waitee `%1%' done; %2% left") % - waitee->name % waitees.size()); - - if (result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure) ++nrFailed; - - if (result == ecNoSubstituters) ++nrNoSubstituters; - - if (result == ecIncompleteClosure) ++nrIncompleteClosure; - - if (waitees.empty() || (result == ecFailed && !settings.keepGoing)) { - - /* If we failed and keepGoing is not set, we remove all - remaining waitees. */ - foreach (Goals::iterator, i, waitees) { - GoalPtr goal = *i; - WeakGoals waiters2; - foreach (WeakGoals::iterator, j, goal->waiters) - if (j->lock() != shared_from_this()) waiters2.push_back(*j); - goal->waiters = waiters2; - } - waitees.clear(); - - worker.wakeUp(shared_from_this()); - } -} - - -void Goal::amDone(ExitCode result) -{ - trace("done"); - assert(exitCode == ecBusy); - assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure); - exitCode = result; - foreach (WeakGoals::iterator, i, waiters) { - GoalPtr goal = i->lock(); - if (goal) goal->waiteeDone(shared_from_this(), result); - } - waiters.clear(); - worker.removeGoal(shared_from_this()); -} - - -void Goal::trace(const format & f) -{ - debug(format("%1%: %2%") % name % f); -} - - - -////////////////////////////////////////////////////////////////////// - - -/* Common initialisation performed in child processes. */ -static void commonChildInit(Pipe & logPipe) -{ - restoreAffinity(); - - /* Put the child in a separate session (and thus a separate - process group) so that it has no controlling terminal (meaning - that e.g. ssh cannot open /dev/tty) and it doesn't receive - terminal signals. */ - if (setsid() == -1) - throw SysError(format("creating a new session")); - - /* Dup the write side of the logger pipe into stderr. */ - if (dup2(logPipe.writeSide, STDERR_FILENO) == -1) - throw SysError("cannot pipe standard error into log file"); - - /* Dup stderr to stdout. */ - if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1) - throw SysError("cannot dup stderr into stdout"); - - /* Reroute stdin to /dev/null. */ - int fdDevNull = open(pathNullDevice.c_str(), O_RDWR); - if (fdDevNull == -1) - throw SysError(format("cannot open `%1%'") % pathNullDevice); - if (dup2(fdDevNull, STDIN_FILENO) == -1) - throw SysError("cannot dup null device into stdin"); - close(fdDevNull); -} - - -/* Convert a string list to an array of char pointers. Careful: the - string list should outlive the array. */ -const char * * strings2CharPtrs(const Strings & ss) -{ - const char * * arr = new const char * [ss.size() + 1]; - const char * * p = arr; - foreach (Strings::const_iterator, i, ss) *p++ = i->c_str(); - *p = 0; - return arr; -} - - -/* Restore default handling of SIGPIPE, otherwise some programs will - randomly say "Broken pipe". */ -static void restoreSIGPIPE() -{ - struct sigaction act, oact; - act.sa_handler = SIG_DFL; - act.sa_flags = 0; - sigemptyset(&act.sa_mask); - if (sigaction(SIGPIPE, &act, &oact)) throw SysError("resetting SIGPIPE"); -} - - -////////////////////////////////////////////////////////////////////// - - -class UserLock -{ -private: - /* POSIX locks suck. If we have a lock on a file, and we open and - close that file again (without closing the original file - descriptor), we lose the lock. So we have to be *very* careful - not to open a lock file on which we are holding a lock. */ - static PathSet lockedPaths; /* !!! not thread-safe */ - - Path fnUserLock; - AutoCloseFD fdUserLock; - - string user; - uid_t uid; - gid_t gid; - -public: - UserLock(); - ~UserLock(); - - void acquire(); - void release(); - - void kill(); - - string getUser() { return user; } - uid_t getUID() { return uid; } - uid_t getGID() { return gid; } - - bool enabled() { return uid != 0; } - -}; - - -PathSet UserLock::lockedPaths; - - -UserLock::UserLock() -{ - uid = gid = 0; -} - - -UserLock::~UserLock() -{ - release(); -} - - -void UserLock::acquire() -{ - assert(uid == 0); - - assert(settings.buildUsersGroup != ""); - - /* Get the members of the build-users-group. */ - struct group * gr = getgrnam(settings.buildUsersGroup.c_str()); - if (!gr) - throw Error(format("the group `%1%' specified in `build-users-group' does not exist") - % settings.buildUsersGroup); - gid = gr->gr_gid; - - /* Copy the result of getgrnam. */ - Strings users; - for (char * * p = gr->gr_mem; *p; ++p) { - debug(format("found build user `%1%'") % *p); - users.push_back(*p); - } - - if (users.empty()) - throw Error(format("the build users group `%1%' has no members") - % settings.buildUsersGroup); - - /* Find a user account that isn't currently in use for another - build. */ - foreach (Strings::iterator, i, users) { - debug(format("trying user `%1%'") % *i); - - struct passwd * pw = getpwnam(i->c_str()); - if (!pw) - throw Error(format("the user `%1%' in the group `%2%' does not exist") - % *i % settings.buildUsersGroup); - - createDirs(settings.nixStateDir + "/userpool"); - - fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str(); - - if (lockedPaths.find(fnUserLock) != lockedPaths.end()) - /* We already have a lock on this one. */ - continue; - - AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT, 0600); - if (fd == -1) - throw SysError(format("opening user lock `%1%'") % fnUserLock); - closeOnExec(fd); - - if (lockFile(fd, ltWrite, false)) { - fdUserLock = fd.borrow(); - lockedPaths.insert(fnUserLock); - user = *i; - uid = pw->pw_uid; - - /* Sanity check... */ - if (uid == getuid() || uid == geteuid()) - throw Error(format("the Nix user should not be a member of `%1%'") - % settings.buildUsersGroup); - - return; - } - } - - throw Error(format("all build users are currently in use; " - "consider creating additional users and adding them to the `%1%' group") - % settings.buildUsersGroup); -} - - -void UserLock::release() -{ - if (uid == 0) return; - fdUserLock.close(); /* releases lock */ - assert(lockedPaths.find(fnUserLock) != lockedPaths.end()); - lockedPaths.erase(fnUserLock); - fnUserLock = ""; - uid = 0; -} - - -void UserLock::kill() -{ - assert(enabled()); - killUser(uid); -} - - -////////////////////////////////////////////////////////////////////// - - -struct HookInstance -{ - /* Pipes for talking to the build hook. */ - Pipe toHook; - - /* Pipe for the hook's standard output/error. */ - Pipe fromHook; - - /* Pipe for the builder's standard output/error. */ - Pipe builderOut; - - /* The process ID of the hook. */ - Pid pid; - - HookInstance(); - - ~HookInstance(); -}; - - -HookInstance::HookInstance() -{ - debug("starting build hook"); - - Path buildHook = absPath(getEnv("NIX_BUILD_HOOK")); - - /* Create a pipe to get the output of the child. */ - fromHook.create(); - - /* Create the communication pipes. */ - toHook.create(); - - /* Create a pipe to get the output of the builder. */ - builderOut.create(); - - /* Fork the hook. */ - pid = maybeVfork(); - switch (pid) { - - case -1: - throw SysError("unable to fork"); - - case 0: - try { /* child */ - - commonChildInit(fromHook); - - if (chdir("/") == -1) throw SysError("changing into `/"); - - /* Dup the communication pipes. */ - if (dup2(toHook.readSide, STDIN_FILENO) == -1) - throw SysError("dupping to-hook read side"); - - /* Use fd 4 for the builder's stdout/stderr. */ - if (dup2(builderOut.writeSide, 4) == -1) - throw SysError("dupping builder's stdout/stderr"); - - execl(buildHook.c_str(), buildHook.c_str(), settings.thisSystem.c_str(), - (format("%1%") % settings.maxSilentTime).str().c_str(), - (format("%1%") % settings.printBuildTrace).str().c_str(), - (format("%1%") % settings.buildTimeout).str().c_str(), - NULL); - - throw SysError(format("executing `%1%'") % buildHook); - - } catch (std::exception & e) { - writeToStderr("build hook error: " + string(e.what()) + "\n"); - } - _exit(1); - } - - /* parent */ - pid.setSeparatePG(true); - pid.setKillSignal(SIGTERM); - fromHook.writeSide.close(); - toHook.readSide.close(); -} - - -HookInstance::~HookInstance() -{ - try { - pid.kill(); - } catch (...) { - ignoreException(); - } -} - - -////////////////////////////////////////////////////////////////////// - - -typedef map<string, string> HashRewrites; - - -string rewriteHashes(string s, const HashRewrites & rewrites) -{ - foreach (HashRewrites::const_iterator, i, rewrites) { - assert(i->first.size() == i->second.size()); - size_t j = 0; - while ((j = s.find(i->first, j)) != string::npos) { - debug(format("rewriting @ %1%") % j); - s.replace(j, i->second.size(), i->second); - } - } - return s; -} - - -////////////////////////////////////////////////////////////////////// - - -typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; - -class SubstitutionGoal; - -class DerivationGoal : public Goal -{ -private: - /* The path of the derivation. */ - Path drvPath; - - /* The specific outputs that we need to build. Empty means all of - them. */ - StringSet wantedOutputs; - - /* Whether additional wanted outputs have been added. */ - bool needRestart; - - /* Whether to retry substituting the outputs after building the - inputs. */ - bool retrySubstitution; - - /* The derivation stored at drvPath. */ - Derivation drv; - - /* The remainder is state held during the build. */ - - /* Locks on the output paths. */ - PathLocks outputLocks; - - /* All input paths (that is, the union of FS closures of the - immediate input paths). */ - PathSet inputPaths; - - /* Referenceable paths (i.e., input and output paths). */ - PathSet allPaths; - - /* Outputs that are already valid. If we're repairing, these are - the outputs that are valid *and* not corrupt. */ - PathSet validPaths; - - /* Outputs that are corrupt or not valid. */ - PathSet missingPaths; - - /* User selected for running the builder. */ - UserLock buildUser; - - /* The process ID of the builder. */ - Pid pid; - - /* The temporary directory. */ - Path tmpDir; - - /* File descriptor for the log file. */ - FILE * fLogFile; - BZFILE * bzLogFile; - AutoCloseFD fdLogFile; - - /* Number of bytes received from the builder's stdout/stderr. */ - unsigned long logSize; - - /* Pipe for the builder's standard output/error. */ - Pipe builderOut; - - /* The build hook. */ - std::shared_ptr<HookInstance> hook; - - /* Whether we're currently doing a chroot build. */ - bool useChroot; - - Path chrootRootDir; - - /* RAII object to delete the chroot directory. */ - std::shared_ptr<AutoDelete> autoDelChroot; - - /* All inputs that are regular files. */ - PathSet regularInputPaths; - - /* Whether this is a fixed-output derivation. */ - bool fixedOutput; - - typedef void (DerivationGoal::*GoalState)(); - GoalState state; - - /* Stuff we need to pass to initChild(). */ - typedef map<Path, Path> DirsInChroot; // maps target path to source path - DirsInChroot dirsInChroot; - typedef map<string, string> Environment; - Environment env; - - /* Hash rewriting. */ - HashRewrites rewritesToTmp, rewritesFromTmp; - typedef map<Path, Path> RedirectedOutputs; - RedirectedOutputs redirectedOutputs; - - BuildMode buildMode; - - /* If we're repairing without a chroot, there may be outputs that - are valid but corrupt. So we redirect these outputs to - temporary paths. */ - PathSet redirectedBadOutputs; - - /* Set of inodes seen during calls to canonicalisePathMetaData() - for this build's outputs. This needs to be shared between - outputs to allow hard links between outputs. */ - InodesSeen inodesSeen; - - /* Magic exit code denoting that setting up the child environment - failed. (It's possible that the child actually returns the - exit code, but ah well.) */ - const static int childSetupFailed = 189; - -public: - DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode = bmNormal); - ~DerivationGoal(); - - void cancel(bool timeout); - - void work(); - - Path getDrvPath() - { - return drvPath; - } - - /* Add wanted outputs to an already existing derivation goal. */ - void addWantedOutputs(const StringSet & outputs); - -private: - /* The states. */ - void init(); - void haveDerivation(); - void outputsSubstituted(); - void closureRepaired(); - void inputsRealised(); - void tryToBuild(); - void buildDone(); - - /* Is the build hook willing to perform the build? */ - HookReply tryBuildHook(); - - /* Start building a derivation. */ - void startBuilder(); - - /* Initialise the builder's process. */ - void initChild(); - - friend int childEntry(void *); - - /* Check that the derivation outputs all exist and register them - as valid. */ - void registerOutputs(); - - /* Open a log file and a pipe to it. */ - Path openLogFile(); - - /* Close the log file. */ - void closeLogFile(); - - /* Delete the temporary directory, if we have one. */ - void deleteTmpDir(bool force); - - /* Callback used by the worker to write to the log. */ - void handleChildOutput(int fd, const string & data); - void handleEOF(int fd); - - /* Return the set of (in)valid paths. */ - PathSet checkPathValidity(bool returnValid, bool checkHash); - - /* Abort the goal if `path' failed to build. */ - bool pathFailed(const Path & path); - - /* Forcibly kill the child process, if any. */ - void killChild(); - - Path addHashRewrite(const Path & path); - - void repairClosure(); -}; - - -DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker) - , wantedOutputs(wantedOutputs) - , needRestart(false) - , retrySubstitution(false) - , fLogFile(0) - , bzLogFile(0) - , useChroot(false) - , buildMode(buildMode) -{ - this->drvPath = drvPath; - state = &DerivationGoal::init; - name = (format("building of `%1%'") % drvPath).str(); - trace("created"); -} - - -DerivationGoal::~DerivationGoal() -{ - /* Careful: we should never ever throw an exception from a - destructor. */ - try { - killChild(); - deleteTmpDir(false); - closeLogFile(); - } catch (...) { - ignoreException(); - } -} - - -void DerivationGoal::killChild() -{ - if (pid != -1) { - worker.childTerminated(pid); - - if (buildUser.enabled()) { - /* If we're using a build user, then there is a tricky - race condition: if we kill the build user before the - child has done its setuid() to the build user uid, then - it won't be killed, and we'll potentially lock up in - pid.wait(). So also send a conventional kill to the - child. */ - ::kill(-pid, SIGKILL); /* ignore the result */ - buildUser.kill(); - pid.wait(true); - } else - pid.kill(); - - assert(pid == -1); - } - - hook.reset(); -} - - -void DerivationGoal::cancel(bool timeout) -{ - if (settings.printBuildTrace && timeout) - printMsg(lvlError, format("@ build-failed %1% - timeout") % drvPath); - killChild(); - amDone(ecFailed); -} - - -void DerivationGoal::work() -{ - (this->*state)(); -} - - -void DerivationGoal::addWantedOutputs(const StringSet & outputs) -{ - /* If we already want all outputs, there is nothing to do. */ - if (wantedOutputs.empty()) return; - - if (outputs.empty()) { - wantedOutputs.clear(); - needRestart = true; - } else - foreach (StringSet::const_iterator, i, outputs) - if (wantedOutputs.find(*i) == wantedOutputs.end()) { - wantedOutputs.insert(*i); - needRestart = true; - } -} - - -void DerivationGoal::init() -{ - trace("init"); - - if (settings.readOnlyMode) - throw Error(format("cannot build derivation `%1%' - no write access to the Nix store") % drvPath); - - /* The first thing to do is to make sure that the derivation - exists. If it doesn't, it may be created through a - substitute. */ - addWaitee(worker.makeSubstitutionGoal(drvPath)); - - state = &DerivationGoal::haveDerivation; -} - - -void DerivationGoal::haveDerivation() -{ - trace("loading derivation"); - - if (nrFailed != 0) { - printMsg(lvlError, format("cannot build missing derivation `%1%'") % drvPath); - amDone(ecFailed); - return; - } - - /* `drvPath' should already be a root, but let's be on the safe - side: if the user forgot to make it a root, we wouldn't want - things being garbage collected while we're busy. */ - worker.store.addTempRoot(drvPath); - - assert(worker.store.isValidPath(drvPath)); - - /* Get the derivation. */ - drv = derivationFromPath(worker.store, drvPath); - - foreach (DerivationOutputs::iterator, i, drv.outputs) - worker.store.addTempRoot(i->second.path); - - /* Check what outputs paths are not already valid. */ - PathSet invalidOutputs = checkPathValidity(false, buildMode == bmRepair); - - /* If they are all valid, then we're done. */ - if (invalidOutputs.size() == 0 && buildMode == bmNormal) { - amDone(ecSuccess); - return; - } - - /* Check whether any output previously failed to build. If so, - don't bother. */ - foreach (PathSet::iterator, i, invalidOutputs) - if (pathFailed(*i)) return; - - /* We are first going to try to create the invalid output paths - through substitutes. If that doesn't work, we'll build - them. */ - if (settings.useSubstitutes && !willBuildLocally(drv)) - foreach (PathSet::iterator, i, invalidOutputs) - addWaitee(worker.makeSubstitutionGoal(*i, buildMode == bmRepair)); - - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - outputsSubstituted(); - else - state = &DerivationGoal::outputsSubstituted; -} - - -void DerivationGoal::outputsSubstituted() -{ - trace("all outputs substituted (maybe)"); - - if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) - throw Error(format("some substitutes for the outputs of derivation `%1%' failed (usually happens due to networking issues); try `--fallback' to build derivation from source ") % drvPath); - - /* If the substitutes form an incomplete closure, then we should - build the dependencies of this derivation, but after that, we - can still use the substitutes for this derivation itself. */ - if (nrIncompleteClosure > 0 && !retrySubstitution) retrySubstitution = true; - - nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; - - if (needRestart) { - needRestart = false; - haveDerivation(); - return; - } - - unsigned int nrInvalid = checkPathValidity(false, buildMode == bmRepair).size(); - if (buildMode == bmNormal && nrInvalid == 0) { - amDone(ecSuccess); - return; - } - if (buildMode == bmRepair && nrInvalid == 0) { - repairClosure(); - return; - } - if (buildMode == bmCheck && nrInvalid > 0) - throw Error(format("some outputs of `%1%' are not valid, so checking is not possible") % drvPath); - - /* Otherwise, at least one of the output paths could not be - produced using a substitute. So we have to build instead. */ - - /* Make sure checkPathValidity() from now on checks all - outputs. */ - wantedOutputs = PathSet(); - - /* The inputs must be built before we can build this goal. */ - foreach (DerivationInputs::iterator, i, drv.inputDrvs) - addWaitee(worker.makeDerivationGoal(i->first, i->second, buildMode == bmRepair ? bmRepair : bmNormal)); - - foreach (PathSet::iterator, i, drv.inputSrcs) - addWaitee(worker.makeSubstitutionGoal(*i)); - - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - inputsRealised(); - else - state = &DerivationGoal::inputsRealised; -} - - -void DerivationGoal::repairClosure() -{ - /* If we're repairing, we now know that our own outputs are valid. - Now check whether the other paths in the outputs closure are - good. If not, then start derivation goals for the derivations - that produced those outputs. */ - - /* Get the output closure. */ - PathSet outputClosure; - foreach (DerivationOutputs::iterator, i, drv.outputs) - computeFSClosure(worker.store, i->second.path, outputClosure); - - /* Filter out our own outputs (which we have already checked). */ - foreach (DerivationOutputs::iterator, i, drv.outputs) - outputClosure.erase(i->second.path); - - /* Get all dependencies of this derivation so that we know which - derivation is responsible for which path in the output - closure. */ - PathSet inputClosure; - computeFSClosure(worker.store, drvPath, inputClosure); - std::map<Path, Path> outputsToDrv; - foreach (PathSet::iterator, i, inputClosure) - if (isDerivation(*i)) { - Derivation drv = derivationFromPath(worker.store, *i); - foreach (DerivationOutputs::iterator, j, drv.outputs) - outputsToDrv[j->second.path] = *i; - } - - /* Check each path (slow!). */ - PathSet broken; - foreach (PathSet::iterator, i, outputClosure) { - if (worker.store.pathContentsGood(*i)) continue; - printMsg(lvlError, format("found corrupted or missing path `%1%' in the output closure of `%2%'") % *i % drvPath); - Path drvPath2 = outputsToDrv[*i]; - if (drvPath2 == "") - addWaitee(worker.makeSubstitutionGoal(*i, true)); - else - addWaitee(worker.makeDerivationGoal(drvPath2, PathSet(), bmRepair)); - } - - if (waitees.empty()) { - amDone(ecSuccess); - return; - } - - state = &DerivationGoal::closureRepaired; -} - - -void DerivationGoal::closureRepaired() -{ - trace("closure repaired"); - if (nrFailed > 0) - throw Error(format("some paths in the output closure of derivation `%1%' could not be repaired") % drvPath); - amDone(ecSuccess); -} - - -void DerivationGoal::inputsRealised() -{ - trace("all inputs realised"); - - if (nrFailed != 0) { - printMsg(lvlError, - format("cannot build derivation `%1%': %2% dependencies couldn't be built") - % drvPath % nrFailed); - amDone(ecFailed); - return; - } - - if (retrySubstitution) { - haveDerivation(); - return; - } - - /* Gather information necessary for computing the closure and/or - running the build hook. */ - - /* The outputs are referenceable paths. */ - foreach (DerivationOutputs::iterator, i, drv.outputs) { - debug(format("building path `%1%'") % i->second.path); - allPaths.insert(i->second.path); - } - - /* Determine the full set of input paths. */ - - /* First, the input derivations. */ - foreach (DerivationInputs::iterator, i, drv.inputDrvs) { - /* Add the relevant output closures of the input derivation - `*i' as input paths. Only add the closures of output paths - that are specified as inputs. */ - assert(worker.store.isValidPath(i->first)); - Derivation inDrv = derivationFromPath(worker.store, i->first); - foreach (StringSet::iterator, j, i->second) - if (inDrv.outputs.find(*j) != inDrv.outputs.end()) - computeFSClosure(worker.store, inDrv.outputs[*j].path, inputPaths); - else - throw Error( - format("derivation `%1%' requires non-existent output `%2%' from input derivation `%3%'") - % drvPath % *j % i->first); - } - - /* Second, the input sources. */ - foreach (PathSet::iterator, i, drv.inputSrcs) - computeFSClosure(worker.store, *i, inputPaths); - - debug(format("added input paths %1%") % showPaths(inputPaths)); - - allPaths.insert(inputPaths.begin(), inputPaths.end()); - - /* Is this a fixed-output derivation? */ - fixedOutput = true; - foreach (DerivationOutputs::iterator, i, drv.outputs) - if (i->second.hash == "") fixedOutput = false; - - /* Okay, try to build. Note that here we don't wait for a build - slot to become available, since we don't need one if there is a - build hook. */ - state = &DerivationGoal::tryToBuild; - worker.wakeUp(shared_from_this()); -} - - -PathSet outputPaths(const DerivationOutputs & outputs) -{ - PathSet paths; - foreach (DerivationOutputs::const_iterator, i, outputs) - paths.insert(i->second.path); - return paths; -} - - -static string get(const StringPairs & map, const string & key) -{ - StringPairs::const_iterator i = map.find(key); - return i == map.end() ? (string) "" : i->second; -} - - -static bool canBuildLocally(const string & platform) -{ - return platform == settings.thisSystem -#ifdef CAN_DO_LINUX32_BUILDS - || (platform == "i686-linux" && settings.thisSystem == "x86_64-linux") -#endif - ; -} - - -bool willBuildLocally(const Derivation & drv) -{ - return get(drv.env, "preferLocalBuild") == "1" && canBuildLocally(drv.platform); -} - - -void DerivationGoal::tryToBuild() -{ - trace("trying to build"); - - /* Check for the possibility that some other goal in this process - has locked the output since we checked in haveDerivation(). - (It can't happen between here and the lockPaths() call below - because we're not allowing multi-threading.) If so, put this - goal to sleep until another goal finishes, then try again. */ - foreach (DerivationOutputs::iterator, i, drv.outputs) - if (pathIsLockedByMe(i->second.path)) { - debug(format("putting derivation `%1%' to sleep because `%2%' is locked by another goal") - % drvPath % i->second.path); - worker.waitForAnyGoal(shared_from_this()); - return; - } - - /* Obtain locks on all output paths. The locks are automatically - released when we exit this function or Nix crashes. If we - can't acquire the lock, then continue; hopefully some other - goal can start a build, and if not, the main loop will sleep a - few seconds and then retry this goal. */ - if (!outputLocks.lockPaths(outputPaths(drv.outputs), "", false)) { - worker.waitForAWhile(shared_from_this()); - return; - } - - /* Now check again whether the outputs are valid. This is because - another process may have started building in parallel. After - it has finished and released the locks, we can (and should) - reuse its results. (Strictly speaking the first check can be - omitted, but that would be less efficient.) Note that since we - now hold the locks on the output paths, no other process can - build this derivation, so no further checks are necessary. */ - validPaths = checkPathValidity(true, buildMode == bmRepair); - assert(buildMode != bmCheck || validPaths.size() == drv.outputs.size()); - if (buildMode != bmCheck && validPaths.size() == drv.outputs.size()) { - debug(format("skipping build of derivation `%1%', someone beat us to it") % drvPath); - outputLocks.setDeletion(true); - amDone(ecSuccess); - return; - } - - missingPaths = outputPaths(drv.outputs); - if (buildMode != bmCheck) - foreach (PathSet::iterator, i, validPaths) missingPaths.erase(*i); - - /* If any of the outputs already exist but are not valid, delete - them. */ - foreach (DerivationOutputs::iterator, i, drv.outputs) { - Path path = i->second.path; - if (worker.store.isValidPath(path)) continue; - if (!pathExists(path)) continue; - debug(format("removing invalid path `%1%'") % path); - deletePath(path); - } - - /* Check again whether any output previously failed to build, - because some other process may have tried and failed before we - acquired the lock. */ - foreach (DerivationOutputs::iterator, i, drv.outputs) - if (pathFailed(i->second.path)) return; - - /* Don't do a remote build if the derivation has the attribute - `preferLocalBuild' set. Also, check and repair modes are only - supported for local builds. */ - bool buildLocally = buildMode != bmNormal || willBuildLocally(drv); - - /* Is the build hook willing to accept this job? */ - if (!buildLocally) { - switch (tryBuildHook()) { - case rpAccept: - /* Yes, it has started doing so. Wait until we get - EOF from the hook. */ - state = &DerivationGoal::buildDone; - return; - case rpPostpone: - /* Not now; wait until at least one child finishes or - the wake-up timeout expires. */ - worker.waitForAWhile(shared_from_this()); - outputLocks.unlock(); - return; - case rpDecline: - /* We should do it ourselves. */ - break; - } - } - - /* Make sure that we are allowed to start a build. If this - derivation prefers to be done locally, do it even if - maxBuildJobs is 0. */ - unsigned int curBuilds = worker.getNrLocalBuilds(); - if (curBuilds >= settings.maxBuildJobs && !(buildLocally && curBuilds == 0)) { - worker.waitForBuildSlot(shared_from_this()); - outputLocks.unlock(); - return; - } - - try { - - /* Okay, we have to build. */ - startBuilder(); - - } catch (BuildError & e) { - printMsg(lvlError, e.msg()); - outputLocks.unlock(); - buildUser.release(); - if (settings.printBuildTrace) - printMsg(lvlError, format("@ build-failed %1% - %2% %3%") - % drvPath % 0 % e.msg()); - worker.permanentFailure = true; - amDone(ecFailed); - return; - } - - /* This state will be reached when we get EOF on the child's - log pipe. */ - state = &DerivationGoal::buildDone; -} - - -void replaceValidPath(const Path & storePath, const Path tmpPath) -{ - /* We can't atomically replace storePath (the original) with - tmpPath (the replacement), so we have to move it out of the - way first. We'd better not be interrupted here, because if - we're repairing (say) Glibc, we end up with a broken system. */ - Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % rand()).str(); - if (pathExists(storePath)) - rename(storePath.c_str(), oldPath.c_str()); - if (rename(tmpPath.c_str(), storePath.c_str()) == -1) - throw SysError(format("moving `%1%' to `%2%'") % tmpPath % storePath); - if (pathExists(oldPath)) - deletePath(oldPath); -} - - -void DerivationGoal::buildDone() -{ - trace("build done"); - - /* Since we got an EOF on the logger pipe, the builder is presumed - to have terminated. In fact, the builder could also have - simply have closed its end of the pipe --- just don't do that - :-) */ - int status; - pid_t savedPid; - if (hook) { - savedPid = hook->pid; - status = hook->pid.wait(true); - } else { - /* !!! this could block! security problem! solution: kill the - child */ - savedPid = pid; - status = pid.wait(true); - } - - debug(format("builder process for `%1%' finished") % drvPath); - - /* So the child is gone now. */ - worker.childTerminated(savedPid); - - /* Close the read side of the logger pipe. */ - if (hook) { - hook->builderOut.readSide.close(); - hook->fromHook.readSide.close(); - } - else builderOut.readSide.close(); - - /* Close the log file. */ - closeLogFile(); - - /* When running under a build user, make sure that all processes - running under that uid are gone. This is to prevent a - malicious user from leaving behind a process that keeps files - open and modifies them after they have been chown'ed to - root. */ - if (buildUser.enabled()) buildUser.kill(); - - bool diskFull = false; - - try { - - /* Check the exit status. */ - if (!statusOk(status)) { - - /* Heuristically check whether the build failure may have - been caused by a disk full condition. We have no way - of knowing whether the build actually got an ENOSPC. - So instead, check if the disk is (nearly) full now. If - so, we don't mark this build as a permanent failure. */ -#if HAVE_STATVFS - unsigned long long required = 8ULL * 1024 * 1024; // FIXME: make configurable - struct statvfs st; - if (statvfs(settings.nixStore.c_str(), &st) == 0 && - (unsigned long long) st.f_bavail * st.f_bsize < required) - diskFull = true; - if (statvfs(tmpDir.c_str(), &st) == 0 && - (unsigned long long) st.f_bavail * st.f_bsize < required) - diskFull = true; -#endif - - deleteTmpDir(false); - - /* Move paths out of the chroot for easier debugging of - build failures. */ - if (useChroot && buildMode == bmNormal) - foreach (PathSet::iterator, i, missingPaths) - if (pathExists(chrootRootDir + *i)) - rename((chrootRootDir + *i).c_str(), i->c_str()); - - if (WIFEXITED(status) && WEXITSTATUS(status) == childSetupFailed) - throw Error(format("failed to set up the build environment for `%1%'") % drvPath); - - if (diskFull) - printMsg(lvlError, "note: build failure may have been caused by lack of free disk space"); - - throw BuildError(format("builder for `%1%' %2%") - % drvPath % statusToString(status)); - } - - /* Compute the FS closure of the outputs and register them as - being valid. */ - registerOutputs(); - - if (buildMode == bmCheck) { - amDone(ecSuccess); - return; - } - - /* Delete unused redirected outputs (when doing hash rewriting). */ - foreach (RedirectedOutputs::iterator, i, redirectedOutputs) - if (pathExists(i->second)) deletePath(i->second); - - /* Delete the chroot (if we were using one). */ - autoDelChroot.reset(); /* this runs the destructor */ - - deleteTmpDir(true); - - /* It is now safe to delete the lock files, since all future - lockers will see that the output paths are valid; they will - not create new lock files with the same names as the old - (unlinked) lock files. */ - outputLocks.setDeletion(true); - outputLocks.unlock(); - - } catch (BuildError & e) { - printMsg(lvlError, e.msg()); - outputLocks.unlock(); - buildUser.release(); - - /* When using a build hook, the hook will return a remote - build failure using exit code 100. Anything else is a hook - problem. */ - bool hookError = hook && - (!WIFEXITED(status) || WEXITSTATUS(status) != 100); - - if (settings.printBuildTrace) { - if (hook && hookError) - printMsg(lvlError, format("@ hook-failed %1% - %2% %3%") - % drvPath % status % e.msg()); - else - printMsg(lvlError, format("@ build-failed %1% - %2% %3%") - % drvPath % 1 % e.msg()); - } - - /* Register the outputs of this build as "failed" so we won't - try to build them again (negative caching). However, don't - do this for fixed-output derivations, since they're likely - to fail for transient reasons (e.g., fetchurl not being - able to access the network). Hook errors (like - communication problems with the remote machine) shouldn't - be cached either. */ - if (settings.cacheFailure && !hookError && !fixedOutput) - foreach (DerivationOutputs::iterator, i, drv.outputs) - worker.store.registerFailedPath(i->second.path); - - worker.permanentFailure = !hookError && !fixedOutput && !diskFull; - amDone(ecFailed); - return; - } - - /* Release the build user, if applicable. */ - buildUser.release(); - - if (settings.printBuildTrace) - printMsg(lvlError, format("@ build-succeeded %1% -") % drvPath); - - amDone(ecSuccess); -} - - -HookReply DerivationGoal::tryBuildHook() -{ - if (!settings.useBuildHook || getEnv("NIX_BUILD_HOOK") == "") return rpDecline; - - if (!worker.hook) - worker.hook = std::shared_ptr<HookInstance>(new HookInstance); - - /* Tell the hook about system features (beyond the system type) - required from the build machine. (The hook could parse the - drv file itself, but this is easier.) */ - Strings features = tokenizeString<Strings>(get(drv.env, "requiredSystemFeatures")); - foreach (Strings::iterator, i, features) checkStoreName(*i); /* !!! abuse */ - - /* Send the request to the hook. */ - writeLine(worker.hook->toHook.writeSide, (format("%1% %2% %3% %4%") - % (worker.getNrLocalBuilds() < settings.maxBuildJobs ? "1" : "0") - % drv.platform % drvPath % concatStringsSep(",", features)).str()); - - /* Read the first line of input, which should be a word indicating - whether the hook wishes to perform the build. */ - string reply; - while (true) { - string s = readLine(worker.hook->fromHook.readSide); - if (string(s, 0, 2) == "# ") { - reply = string(s, 2); - break; - } - s += "\n"; - writeToStderr(s); - } - - debug(format("hook reply is `%1%'") % reply); - - if (reply == "decline" || reply == "postpone") - return reply == "decline" ? rpDecline : rpPostpone; - else if (reply != "accept") - throw Error(format("bad hook reply `%1%'") % reply); - - printMsg(lvlTalkative, format("using hook to build path(s) %1%") % showPaths(missingPaths)); - - hook = worker.hook; - worker.hook.reset(); - - /* Tell the hook all the inputs that have to be copied to the - remote system. This unfortunately has to contain the entire - derivation closure to ensure that the validity invariant holds - on the remote system. (I.e., it's unfortunate that we have to - list it since the remote system *probably* already has it.) */ - PathSet allInputs; - allInputs.insert(inputPaths.begin(), inputPaths.end()); - computeFSClosure(worker.store, drvPath, allInputs); - - string s; - foreach (PathSet::iterator, i, allInputs) { s += *i; s += ' '; } - writeLine(hook->toHook.writeSide, s); - - /* Tell the hooks the missing outputs that have to be copied back - from the remote system. */ - s = ""; - foreach (PathSet::iterator, i, missingPaths) { s += *i; s += ' '; } - writeLine(hook->toHook.writeSide, s); - - hook->toHook.writeSide.close(); - - /* Create the log file and pipe. */ - Path logFile = openLogFile(); - - set<int> fds; - fds.insert(hook->fromHook.readSide); - fds.insert(hook->builderOut.readSide); - worker.childStarted(shared_from_this(), hook->pid, fds, false, false); - - if (settings.printBuildTrace) - printMsg(lvlError, format("@ build-started %1% - %2% %3%") - % drvPath % drv.platform % logFile); - - return rpAccept; -} - - -void chmod_(const Path & path, mode_t mode) -{ - if (chmod(path.c_str(), mode) == -1) - throw SysError(format("setting permissions on `%1%'") % path); -} - - -int childEntry(void * arg) -{ - ((DerivationGoal *) arg)->initChild(); - return 1; -} - - -void DerivationGoal::startBuilder() -{ - startNest(nest, lvlInfo, format( - buildMode == bmRepair ? "repairing path(s) %1%" : - buildMode == bmCheck ? "checking path(s) %1%" : - "building path(s) %1%") % showPaths(missingPaths)); - - /* Right platform? */ - if (!canBuildLocally(drv.platform)) { - if (settings.printBuildTrace) - printMsg(lvlError, format("@ unsupported-platform %1% %2%") % drvPath % drv.platform); - throw Error( - format("a `%1%' is required to build `%3%', but I am a `%2%'") - % drv.platform % settings.thisSystem % drvPath); - } - - /* Construct the environment passed to the builder. */ - - /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when - PATH is not set. We don't want this, so we fill it in with some dummy - value. */ - env["PATH"] = "/path-not-set"; - - /* Set HOME to a non-existing path to prevent certain programs from using - /etc/passwd (or NIS, or whatever) to locate the home directory (for - example, wget looks for ~/.wgetrc). I.e., these tools use /etc/passwd - if HOME is not set, but they will just assume that the settings file - they are looking for does not exist if HOME is set but points to some - non-existing path. */ - Path homeDir = "/homeless-shelter"; - env["HOME"] = homeDir; - - /* Tell the builder where the Nix store is. Usually they - shouldn't care, but this is useful for purity checking (e.g., - the compiler or linker might only want to accept paths to files - in the store or in the build directory). */ - env["NIX_STORE"] = settings.nixStore; - - /* The maximum number of cores to utilize for parallel building. */ - env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str(); - - /* Add all bindings specified in the derivation. */ - foreach (StringPairs::iterator, i, drv.env) - env[i->first] = i->second; - - /* Create a temporary directory where the build will take - place. */ - tmpDir = createTempDir("", "nix-build-" + storePathToName(drvPath), false, false, 0700); - - /* For convenience, set an environment pointing to the top build - directory. */ - env["NIX_BUILD_TOP"] = tmpDir; - - /* Also set TMPDIR and variants to point to this directory. */ - env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDir; - - /* Explicitly set PWD to prevent problems with chroot builds. In - particular, dietlibc cannot figure out the cwd because the - inode of the current directory doesn't appear in .. (because - getdents returns the inode of the mount point). */ - env["PWD"] = tmpDir; - - /* Compatibility hack with Nix <= 0.7: if this is a fixed-output - derivation, tell the builder, so that for instance `fetchurl' - can skip checking the output. On older Nixes, this environment - variable won't be set, so `fetchurl' will do the check. */ - if (fixedOutput) env["NIX_OUTPUT_CHECKED"] = "1"; - - /* *Only* if this is a fixed-output derivation, propagate the - values of the environment variables specified in the - `impureEnvVars' attribute to the builder. This allows for - instance environment variables for proxy configuration such as - `http_proxy' to be easily passed to downloaders like - `fetchurl'. Passing such environment variables from the caller - to the builder is generally impure, but the output of - fixed-output derivations is by definition pure (since we - already know the cryptographic hash of the output). */ - if (fixedOutput) { - Strings varNames = tokenizeString<Strings>(get(drv.env, "impureEnvVars")); - foreach (Strings::iterator, i, varNames) env[*i] = getEnv(*i); - } - - /* The `exportReferencesGraph' feature allows the references graph - to be passed to a builder. This attribute should be a list of - pairs [name1 path1 name2 path2 ...]. The references graph of - each `pathN' will be stored in a text file `nameN' in the - temporary build directory. The text files have the format used - by `nix-store --register-validity'. However, the deriver - fields are left empty. */ - string s = get(drv.env, "exportReferencesGraph"); - Strings ss = tokenizeString<Strings>(s); - if (ss.size() % 2 != 0) - throw BuildError(format("odd number of tokens in `exportReferencesGraph': `%1%'") % s); - for (Strings::iterator i = ss.begin(); i != ss.end(); ) { - string fileName = *i++; - checkStoreName(fileName); /* !!! abuse of this function */ - - /* Check that the store path is valid. */ - Path storePath = *i++; - if (!isInStore(storePath)) - throw BuildError(format("`exportReferencesGraph' contains a non-store path `%1%'") - % storePath); - storePath = toStorePath(storePath); - if (!worker.store.isValidPath(storePath)) - throw BuildError(format("`exportReferencesGraph' contains an invalid path `%1%'") - % storePath); - - /* If there are derivations in the graph, then include their - outputs as well. This is useful if you want to do things - like passing all build-time dependencies of some path to a - derivation that builds a NixOS DVD image. */ - PathSet paths, paths2; - computeFSClosure(worker.store, storePath, paths); - paths2 = paths; - - foreach (PathSet::iterator, j, paths2) { - if (isDerivation(*j)) { - Derivation drv = derivationFromPath(worker.store, *j); - foreach (DerivationOutputs::iterator, k, drv.outputs) - computeFSClosure(worker.store, k->second.path, paths); - } - } - - /* Write closure info to `fileName'. */ - writeFile(tmpDir + "/" + fileName, - worker.store.makeValidityRegistration(paths, false, false)); - } - - - /* If `build-users-group' is not empty, then we have to build as - one of the members of that group. */ - if (settings.buildUsersGroup != "") { - buildUser.acquire(); - assert(buildUser.getUID() != 0); - assert(buildUser.getGID() != 0); - - /* Make sure that no other processes are executing under this - uid. */ - buildUser.kill(); - - /* Change ownership of the temporary build directory. */ - if (chown(tmpDir.c_str(), buildUser.getUID(), buildUser.getGID()) == -1) - throw SysError(format("cannot change ownership of `%1%'") % tmpDir); - - /* Check that the Nix store has the appropriate permissions, - i.e., owned by root and mode 1775 (sticky bit on so that - the builder can create its output but not mess with the - outputs of other processes). */ - struct stat st; - if (stat(settings.nixStore.c_str(), &st) == -1) - throw SysError(format("cannot stat `%1%'") % settings.nixStore); - if (!(st.st_mode & S_ISVTX) || - ((st.st_mode & S_IRWXG) != S_IRWXG) || - (st.st_gid != buildUser.getGID())) - throw Error(format( - "builder does not have write permission to `%2%'; " - "try `chgrp %1% %2%; chmod 1775 %2%'") - % buildUser.getGID() % settings.nixStore); - } - - - /* Are we doing a chroot build? Note that fixed-output - derivations are never done in a chroot, mainly so that - functions like fetchurl (which needs a proper /etc/resolv.conf) - work properly. Purity checking for fixed-output derivations - is somewhat pointless anyway. */ - useChroot = settings.useChroot; - - if (fixedOutput) useChroot = false; - - /* Hack to allow derivations to disable chroot builds. */ - if (get(drv.env, "__noChroot") == "1") useChroot = false; - - if (useChroot) { -#if CHROOT_ENABLED - /* Create a temporary directory in which we set up the chroot - environment using bind-mounts. We put it in the Nix store - to ensure that we can create hard-links to non-directory - inputs in the fake Nix store in the chroot (see below). */ - chrootRootDir = drvPath + ".chroot"; - if (pathExists(chrootRootDir)) deletePath(chrootRootDir); - - /* Clean up the chroot directory automatically. */ - autoDelChroot = std::shared_ptr<AutoDelete>(new AutoDelete(chrootRootDir)); - - printMsg(lvlChatty, format("setting up chroot environment in `%1%'") % chrootRootDir); - - /* Create a writable /tmp in the chroot. Many builders need - this. (Of course they should really respect $TMPDIR - instead.) */ - Path chrootTmpDir = chrootRootDir + "/tmp"; - createDirs(chrootTmpDir); - chmod_(chrootTmpDir, 01777); - - /* Create a /etc/passwd with entries for the build user and the - nobody account. The latter is kind of a hack to support - Samba-in-QEMU. */ - createDirs(chrootRootDir + "/etc"); - - writeFile(chrootRootDir + "/etc/passwd", - (format( - "nixbld:x:%1%:%2%:Nix build user:/:/noshell\n" - "nobody:x:65534:65534:Nobody:/:/noshell\n") - % (buildUser.enabled() ? buildUser.getUID() : getuid()) - % (buildUser.enabled() ? buildUser.getGID() : getgid())).str()); - - /* Declare the build user's group so that programs get a consistent - view of the system (e.g., "id -gn"). */ - writeFile(chrootRootDir + "/etc/group", - (format("nixbld:!:%1%:\n") - % (buildUser.enabled() ? buildUser.getGID() : getgid())).str()); - - /* Create /etc/hosts with localhost entry. */ - writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n"); - - /* Bind-mount a user-configurable set of directories from the - host file system. */ - foreach (StringSet::iterator, i, settings.dirsInChroot) { - size_t p = i->find('='); - if (p == string::npos) - dirsInChroot[*i] = *i; - else - dirsInChroot[string(*i, 0, p)] = string(*i, p + 1); - } - dirsInChroot[tmpDir] = tmpDir; - - /* Make the closure of the inputs available in the chroot, - rather than the whole Nix store. This prevents any access - to undeclared dependencies. Directories are bind-mounted, - while other inputs are hard-linked (since only directories - can be bind-mounted). !!! As an extra security - precaution, make the fake Nix store only writable by the - build user. */ - createDirs(chrootRootDir + settings.nixStore); - chmod_(chrootRootDir + settings.nixStore, 01777); - - foreach (PathSet::iterator, i, inputPaths) { - struct stat st; - if (lstat(i->c_str(), &st)) - throw SysError(format("getting attributes of path `%1%'") % *i); - if (S_ISDIR(st.st_mode)) - dirsInChroot[*i] = *i; - else { - Path p = chrootRootDir + *i; - if (link(i->c_str(), p.c_str()) == -1) { - /* Hard-linking fails if we exceed the maximum - link count on a file (e.g. 32000 of ext3), - which is quite possible after a `nix-store - --optimise'. */ - if (errno != EMLINK) - throw SysError(format("linking `%1%' to `%2%'") % p % *i); - StringSink sink; - dumpPath(*i, sink); - StringSource source(sink.s); - restorePath(p, source); - } - - regularInputPaths.insert(*i); - } - } - - /* If we're repairing or checking, it's possible that we're - rebuilding a path that is in settings.dirsInChroot - (typically the dependencies of /bin/sh). Throw them - out. */ - if (buildMode != bmNormal) - foreach (DerivationOutputs::iterator, i, drv.outputs) - dirsInChroot.erase(i->second.path); - -#else - throw Error("chroot builds are not supported on this platform"); -#endif - } - - else { - - if (pathExists(homeDir)) - throw Error(format("directory `%1%' exists; please remove it") % homeDir); - - /* We're not doing a chroot build, but we have some valid - output paths. Since we can't just overwrite or delete - them, we have to do hash rewriting: i.e. in the - environment/arguments passed to the build, we replace the - hashes of the valid outputs with unique dummy strings; - after the build, we discard the redirected outputs - corresponding to the valid outputs, and rewrite the - contents of the new outputs to replace the dummy strings - with the actual hashes. */ - if (validPaths.size() > 0) - foreach (PathSet::iterator, i, validPaths) - addHashRewrite(*i); - - /* If we're repairing, then we don't want to delete the - corrupt outputs in advance. So rewrite them as well. */ - if (buildMode == bmRepair) - foreach (PathSet::iterator, i, missingPaths) - if (worker.store.isValidPath(*i) && pathExists(*i)) { - addHashRewrite(*i); - redirectedBadOutputs.insert(*i); - } - } - - - /* Run the builder. */ - printMsg(lvlChatty, format("executing builder `%1%'") % drv.builder); - - /* Create the log file. */ - Path logFile = openLogFile(); - - /* Create a pipe to get the output of the builder. */ - builderOut.create(); - - /* Fork a child to build the package. Note that while we - currently use forks to run and wait for the children, it - shouldn't be hard to use threads for this on systems where - fork() is unavailable or inefficient. - - If we're building in a chroot, then also set up private - namespaces for the build: - - - The PID namespace causes the build to start as PID 1. - Processes outside of the chroot are not visible to those on - the inside, but processes inside the chroot are visible from - the outside (though with different PIDs). - - - The private mount namespace ensures that all the bind mounts - we do will only show up in this process and its children, and - will disappear automatically when we're done. - - - The private network namespace ensures that the builder cannot - talk to the outside world (or vice versa). It only has a - private loopback interface. - - - The IPC namespace prevents the builder from communicating - with outside processes using SysV IPC mechanisms (shared - memory, message queues, semaphores). It also ensures that - all IPC objects are destroyed when the builder exits. - - - The UTS namespace ensures that builders see a hostname of - localhost rather than the actual hostname. - */ -#if CHROOT_ENABLED - if (useChroot) { - char stack[32 * 1024]; - pid = clone(childEntry, stack + sizeof(stack) - 8, - CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWNET | CLONE_NEWIPC | CLONE_NEWUTS | SIGCHLD, this); - } else -#endif - { - pid = fork(); - if (pid == 0) initChild(); - } - - if (pid == -1) throw SysError("unable to fork"); - - /* parent */ - pid.setSeparatePG(true); - builderOut.writeSide.close(); - worker.childStarted(shared_from_this(), pid, - singleton<set<int> >(builderOut.readSide), true, true); - - if (settings.printBuildTrace) { - printMsg(lvlError, format("@ build-started %1% - %2% %3%") - % drvPath % drv.platform % logFile); - } -} - - -void DerivationGoal::initChild() -{ - /* Warning: in the child we should absolutely not make any SQLite - calls! */ - - bool inSetup = true; - - try { /* child */ - -#if CHROOT_ENABLED - if (useChroot) { - /* Initialise the loopback interface. */ - AutoCloseFD fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP)); - if (fd == -1) throw SysError("cannot open IP socket"); - - struct ifreq ifr; - strcpy(ifr.ifr_name, "lo"); - ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING; - if (ioctl(fd, SIOCSIFFLAGS, &ifr) == -1) - throw SysError("cannot set loopback interface flags"); - - fd.close(); - - /* Set the hostname etc. to fixed values. */ - char hostname[] = "localhost"; - sethostname(hostname, sizeof(hostname)); - char domainname[] = "(none)"; // kernel default - setdomainname(domainname, sizeof(domainname)); - - /* Make all filesystems private. This is necessary - because subtrees may have been mounted as "shared" - (MS_SHARED). (Systemd does this, for instance.) Even - though we have a private mount namespace, mounting - filesystems on top of a shared subtree still propagates - outside of the namespace. Making a subtree private is - local to the namespace, though, so setting MS_PRIVATE - does not affect the outside world. */ - Strings mounts = tokenizeString<Strings>(readFile("/proc/self/mountinfo", true), "\n"); - foreach (Strings::iterator, i, mounts) { - vector<string> fields = tokenizeString<vector<string> >(*i, " "); - string fs = decodeOctalEscaped(fields.at(4)); - if (mount(0, fs.c_str(), 0, MS_PRIVATE, 0) == -1) - throw SysError(format("unable to make filesystem `%1%' private") % fs); - } - - /* Set up a nearly empty /dev, unless the user asked to - bind-mount the host /dev. */ - if (dirsInChroot.find("/dev") == dirsInChroot.end()) { - createDirs(chrootRootDir + "/dev/shm"); - createDirs(chrootRootDir + "/dev/pts"); - Strings ss; - ss.push_back("/dev/full"); -#ifdef __linux__ - if (pathExists("/dev/kvm")) - ss.push_back("/dev/kvm"); -#endif - ss.push_back("/dev/null"); - ss.push_back("/dev/random"); - ss.push_back("/dev/tty"); - ss.push_back("/dev/urandom"); - ss.push_back("/dev/zero"); - foreach (Strings::iterator, i, ss) dirsInChroot[*i] = *i; - createSymlink("/proc/self/fd", chrootRootDir + "/dev/fd"); - createSymlink("/proc/self/fd/0", chrootRootDir + "/dev/stdin"); - createSymlink("/proc/self/fd/1", chrootRootDir + "/dev/stdout"); - createSymlink("/proc/self/fd/2", chrootRootDir + "/dev/stderr"); - } - - /* Bind-mount all the directories from the "host" - filesystem that we want in the chroot - environment. */ - foreach (DirsInChroot::iterator, i, dirsInChroot) { - struct stat st; - Path source = i->second; - Path target = chrootRootDir + i->first; - if (source == "/proc") continue; // backwards compatibility - debug(format("bind mounting `%1%' to `%2%'") % source % target); - if (stat(source.c_str(), &st) == -1) - throw SysError(format("getting attributes of path `%1%'") % source); - if (S_ISDIR(st.st_mode)) - createDirs(target); - else { - createDirs(dirOf(target)); - writeFile(target, ""); - } - if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1) - throw SysError(format("bind mount from `%1%' to `%2%' failed") % source % target); - } - - /* Bind a new instance of procfs on /proc to reflect our - private PID namespace. */ - createDirs(chrootRootDir + "/proc"); - if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1) - throw SysError("mounting /proc"); - - /* Mount a new tmpfs on /dev/shm to ensure that whatever - the builder puts in /dev/shm is cleaned up automatically. */ - if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0, 0) == -1) - throw SysError("mounting /dev/shm"); - - /* Mount a new devpts on /dev/pts. Note that this - requires the kernel to be compiled with - CONFIG_DEVPTS_MULTIPLE_INSTANCES=y (which is the case - if /dev/ptx/ptmx exists). */ - if (pathExists("/dev/pts/ptmx") && - !pathExists(chrootRootDir + "/dev/ptmx") - && dirsInChroot.find("/dev/pts") == dirsInChroot.end()) - { - if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == -1) - throw SysError("mounting /dev/pts"); - createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx"); - - /* Make sure /dev/pts/ptmx is world-writable. With some - Linux versions, it is created with permissions 0. */ - chmod_(chrootRootDir + "/dev/pts/ptmx", 0666); - } - - /* Do the chroot(). Below we do a chdir() to the - temporary build directory to make sure the current - directory is in the chroot. (Actually the order - doesn't matter, since due to the bind mount tmpDir and - tmpRootDit/tmpDir are the same directories.) */ - if (chroot(chrootRootDir.c_str()) == -1) - throw SysError(format("cannot change root directory to `%1%'") % chrootRootDir); - } -#endif - - commonChildInit(builderOut); - - if (chdir(tmpDir.c_str()) == -1) - throw SysError(format("changing into `%1%'") % tmpDir); - - /* Close all other file descriptors. */ - closeMostFDs(set<int>()); - -#ifdef CAN_DO_LINUX32_BUILDS - /* Change the personality to 32-bit if we're doing an - i686-linux build on an x86_64-linux machine. */ - struct utsname utsbuf; - uname(&utsbuf); - if (drv.platform == "i686-linux" && - (settings.thisSystem == "x86_64-linux" || - (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) { - if (personality(0x0008 | 0x8000000 /* == PER_LINUX32_3GB */) == -1) - throw SysError("cannot set i686-linux personality"); - } - - /* Impersonate a Linux 2.6 machine to get some determinism in - builds that depend on the kernel version. */ - if ((drv.platform == "i686-linux" || drv.platform == "x86_64-linux") && settings.impersonateLinux26) { - int cur = personality(0xffffffff); - if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */); - } -#endif - - /* Fill in the environment. */ - Strings envStrs; - foreach (Environment::const_iterator, i, env) - envStrs.push_back(rewriteHashes(i->first + "=" + i->second, rewritesToTmp)); - const char * * envArr = strings2CharPtrs(envStrs); - - Path program = drv.builder.c_str(); - std::vector<const char *> args; /* careful with c_str()! */ - string user; /* must be here for its c_str()! */ - - /* If we are running in `build-users' mode, then switch to the - user we allocated above. Make sure that we drop all root - privileges. Note that above we have closed all file - descriptors except std*, so that's safe. Also note that - setuid() when run as root sets the real, effective and - saved UIDs. */ - if (buildUser.enabled()) { - printMsg(lvlChatty, format("switching to user `%1%'") % buildUser.getUser()); - - if (setgroups(0, 0) == -1) - throw SysError("cannot clear the set of supplementary groups"); - - if (setgid(buildUser.getGID()) == -1 || - getgid() != buildUser.getGID() || - getegid() != buildUser.getGID()) - throw SysError("setgid failed"); - - if (setuid(buildUser.getUID()) == -1 || - getuid() != buildUser.getUID() || - geteuid() != buildUser.getUID()) - throw SysError("setuid failed"); - } - - /* Fill in the arguments. */ - string builderBasename = baseNameOf(drv.builder); - args.push_back(builderBasename.c_str()); - foreach (Strings::iterator, i, drv.args) - args.push_back(rewriteHashes(*i, rewritesToTmp).c_str()); - args.push_back(0); - - restoreSIGPIPE(); - - /* Execute the program. This should not return. */ - inSetup = false; - execve(program.c_str(), (char * *) &args[0], (char * *) envArr); - - throw SysError(format("executing `%1%'") % drv.builder); - - } catch (std::exception & e) { - writeToStderr("build error: " + string(e.what()) + "\n"); - _exit(inSetup ? childSetupFailed : 1); - } - - abort(); /* never reached */ -} - - -/* Parse a list of reference specifiers. Each element must either be - a store path, or the symbolic name of the output of the derivation - (such as `out'). */ -PathSet parseReferenceSpecifiers(const Derivation & drv, string attr) -{ - PathSet result; - Paths paths = tokenizeString<Paths>(attr); - foreach (Strings::iterator, i, paths) { - if (isStorePath(*i)) - result.insert(*i); - else if (drv.outputs.find(*i) != drv.outputs.end()) - result.insert(drv.outputs.find(*i)->second.path); - else throw BuildError( - format("derivation contains an illegal reference specifier `%1%'") - % *i); - } - return result; -} - - -void DerivationGoal::registerOutputs() -{ - /* When using a build hook, the build hook can register the output - as valid (by doing `nix-store --import'). If so we don't have - to do anything here. */ - if (hook) { - bool allValid = true; - foreach (DerivationOutputs::iterator, i, drv.outputs) - if (!worker.store.isValidPath(i->second.path)) allValid = false; - if (allValid) return; - } - - ValidPathInfos infos; - - /* Check whether the output paths were created, and grep each - output path to determine what other paths it references. Also make all - output paths read-only. */ - foreach (DerivationOutputs::iterator, i, drv.outputs) { - Path path = i->second.path; - if (missingPaths.find(path) == missingPaths.end()) continue; - - Path actualPath = path; - if (useChroot) { - actualPath = chrootRootDir + path; - if (pathExists(actualPath)) { - /* Move output paths from the chroot to the Nix store. */ - if (buildMode == bmRepair) - replaceValidPath(path, actualPath); - else - if (buildMode != bmCheck && rename(actualPath.c_str(), path.c_str()) == -1) - throw SysError(format("moving build output `%1%' from the chroot to the Nix store") % path); - } - if (buildMode != bmCheck) actualPath = path; - } else { - Path redirected = redirectedOutputs[path]; - if (buildMode == bmRepair - && redirectedBadOutputs.find(path) != redirectedBadOutputs.end() - && pathExists(redirected)) - replaceValidPath(path, redirected); - if (buildMode == bmCheck) - actualPath = redirected; - } - - struct stat st; - if (lstat(actualPath.c_str(), &st) == -1) { - if (errno == ENOENT) - throw BuildError( - format("builder for `%1%' failed to produce output path `%2%'") - % drvPath % path); - throw SysError(format("getting attributes of path `%1%'") % actualPath); - } - -#ifndef __CYGWIN__ - /* Check that the output is not group or world writable, as - that means that someone else can have interfered with the - build. Also, the output should be owned by the build - user. */ - if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) || - (buildUser.enabled() && st.st_uid != buildUser.getUID())) - throw BuildError(format("suspicious ownership or permission on `%1%'; rejecting this build output") % path); -#endif - - /* Apply hash rewriting if necessary. */ - bool rewritten = false; - if (!rewritesFromTmp.empty()) { - printMsg(lvlError, format("warning: rewriting hashes in `%1%'; cross fingers") % path); - - /* Canonicalise first. This ensures that the path we're - rewriting doesn't contain a hard link to /etc/shadow or - something like that. */ - canonicalisePathMetaData(actualPath, buildUser.enabled() ? buildUser.getUID() : -1, inodesSeen); - - /* FIXME: this is in-memory. */ - StringSink sink; - dumpPath(actualPath, sink); - deletePath(actualPath); - sink.s = rewriteHashes(sink.s, rewritesFromTmp); - StringSource source(sink.s); - restorePath(actualPath, source); - - rewritten = true; - } - - startNest(nest, lvlTalkative, - format("scanning for references inside `%1%'") % path); - - /* Check that fixed-output derivations produced the right - outputs (i.e., the content hash should match the specified - hash). */ - if (i->second.hash != "") { - - bool recursive; HashType ht; Hash h; - i->second.parseHashInfo(recursive, ht, h); - - if (!recursive) { - /* The output path should be a regular file without - execute permission. */ - if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) - throw BuildError( - format("output path `%1% should be a non-executable regular file") % path); - } - - /* Check the hash. */ - Hash h2 = recursive ? hashPath(ht, actualPath).first : hashFile(ht, actualPath); - if (h != h2) - throw BuildError( - format("output path `%1%' should have %2% hash `%3%', instead has `%4%'") - % path % i->second.hashAlgo % printHash16or32(h) % printHash16or32(h2)); - } - - /* Get rid of all weird permissions. This also checks that - all files are owned by the build user, if applicable. */ - canonicalisePathMetaData(actualPath, - buildUser.enabled() && !rewritten ? buildUser.getUID() : -1, inodesSeen); - - /* For this output path, find the references to other paths - contained in it. Compute the SHA-256 NAR hash at the same - time. The hash is stored in the database so that we can - verify later on whether nobody has messed with the store. */ - HashResult hash; - PathSet references = scanForReferences(actualPath, allPaths, hash); - - if (buildMode == bmCheck) { - ValidPathInfo info = worker.store.queryPathInfo(path); - if (hash.first != info.hash) - throw Error(format("derivation `%2%' may not be deterministic: hash mismatch in output `%1%'") % drvPath % path); - continue; - } - - /* For debugging, print out the referenced and unreferenced - paths. */ - foreach (PathSet::iterator, i, inputPaths) { - PathSet::iterator j = references.find(*i); - if (j == references.end()) - debug(format("unreferenced input: `%1%'") % *i); - else - debug(format("referenced input: `%1%'") % *i); - } - - /* If the derivation specifies an `allowedReferences' - attribute (containing a list of paths that the output may - refer to), check that all references are in that list. !!! - allowedReferences should really be per-output. */ - if (drv.env.find("allowedReferences") != drv.env.end()) { - PathSet allowed = parseReferenceSpecifiers(drv, get(drv.env, "allowedReferences")); - foreach (PathSet::iterator, i, references) - if (allowed.find(*i) == allowed.end()) - throw BuildError(format("output is not allowed to refer to path `%1%'") % *i); - } - - worker.store.optimisePath(path); // FIXME: combine with scanForReferences() - - worker.store.markContentsGood(path); - - ValidPathInfo info; - info.path = path; - info.hash = hash.first; - info.narSize = hash.second; - info.references = references; - info.deriver = drvPath; - infos.push_back(info); - } - - if (buildMode == bmCheck) return; - - /* Register each output path as valid, and register the sets of - paths referenced by each of them. If there are cycles in the - outputs, this will fail. */ - worker.store.registerValidPaths(infos); -} - - -string drvsLogDir = "drvs"; - - -Path DerivationGoal::openLogFile() -{ - logSize = 0; - - if (!settings.keepLog) return ""; - - string baseName = baseNameOf(drvPath); - - /* Create a log file. */ - Path dir = (format("%1%/%2%/%3%/") % settings.nixLogDir % drvsLogDir % string(baseName, 0, 2)).str(); - createDirs(dir); - - if (settings.compressLog) { - - Path logFileName = (format("%1%/%2%.bz2") % dir % string(baseName, 2)).str(); - AutoCloseFD fd = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666); - if (fd == -1) throw SysError(format("creating log file `%1%'") % logFileName); - closeOnExec(fd); - - if (!(fLogFile = fdopen(fd.borrow(), "w"))) - throw SysError(format("opening file `%1%'") % logFileName); - - int err; - if (!(bzLogFile = BZ2_bzWriteOpen(&err, fLogFile, 9, 0, 0))) - throw Error(format("cannot open compressed log file `%1%'") % logFileName); - - return logFileName; - - } else { - Path logFileName = (format("%1%/%2%") % dir % string(baseName, 2)).str(); - fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666); - if (fdLogFile == -1) throw SysError(format("creating log file `%1%'") % logFileName); - closeOnExec(fdLogFile); - return logFileName; - } -} - - -void DerivationGoal::closeLogFile() -{ - if (bzLogFile) { - int err; - BZ2_bzWriteClose(&err, bzLogFile, 0, 0, 0); - bzLogFile = 0; - if (err != BZ_OK) throw Error(format("cannot close compressed log file (BZip2 error = %1%)") % err); - } - - if (fLogFile) { - fclose(fLogFile); - fLogFile = 0; - } - - fdLogFile.close(); -} - - -void DerivationGoal::deleteTmpDir(bool force) -{ - if (tmpDir != "") { - if (settings.keepFailed && !force) { - printMsg(lvlError, - format("note: keeping build directory `%2%'") - % drvPath % tmpDir); - chmod(tmpDir.c_str(), 0755); - } - else - deletePath(tmpDir); - tmpDir = ""; - } -} - - -void DerivationGoal::handleChildOutput(int fd, const string & data) -{ - if ((hook && fd == hook->builderOut.readSide) || - (!hook && fd == builderOut.readSide)) - { - logSize += data.size(); - if (settings.maxLogSize && logSize > settings.maxLogSize) { - printMsg(lvlError, - format("%1% killed after writing more than %2% bytes of log output") - % getName() % settings.maxLogSize); - cancel(true); // not really a timeout, but close enough - return; - } - if (verbosity >= settings.buildVerbosity) - writeToStderr(data); - if (bzLogFile) { - int err; - BZ2_bzWrite(&err, bzLogFile, (unsigned char *) data.data(), data.size()); - if (err != BZ_OK) throw Error(format("cannot write to compressed log file (BZip2 error = %1%)") % err); - } else if (fdLogFile != -1) - writeFull(fdLogFile, (unsigned char *) data.data(), data.size()); - } - - if (hook && fd == hook->fromHook.readSide) - writeToStderr(data); -} - - -void DerivationGoal::handleEOF(int fd) -{ - worker.wakeUp(shared_from_this()); -} - - -PathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash) -{ - PathSet result; - foreach (DerivationOutputs::iterator, i, drv.outputs) { - if (!wantOutput(i->first, wantedOutputs)) continue; - bool good = - worker.store.isValidPath(i->second.path) && - (!checkHash || worker.store.pathContentsGood(i->second.path)); - if (good == returnValid) result.insert(i->second.path); - } - return result; -} - - -bool DerivationGoal::pathFailed(const Path & path) -{ - if (!settings.cacheFailure) return false; - - if (!worker.store.hasPathFailed(path)) return false; - - printMsg(lvlError, format("builder for `%1%' failed previously (cached)") % path); - - if (settings.printBuildTrace) - printMsg(lvlError, format("@ build-failed %1% - cached") % drvPath); - - worker.permanentFailure = true; - amDone(ecFailed); - - return true; -} - - -Path DerivationGoal::addHashRewrite(const Path & path) -{ - string h1 = string(path, settings.nixStore.size() + 1, 32); - string h2 = string(printHash32(hashString(htSHA256, "rewrite:" + drvPath + ":" + path)), 0, 32); - Path p = settings.nixStore + "/" + h2 + string(path, settings.nixStore.size() + 33); - if (pathExists(p)) deletePath(p); - assert(path.size() == p.size()); - rewritesToTmp[h1] = h2; - rewritesFromTmp[h2] = h1; - redirectedOutputs[path] = p; - return p; -} - - -////////////////////////////////////////////////////////////////////// - - -class SubstitutionGoal : public Goal -{ - friend class Worker; - -private: - /* The store path that should be realised through a substitute. */ - Path storePath; - - /* The remaining substituters. */ - Paths subs; - - /* The current substituter. */ - Path sub; - - /* Whether any substituter can realise this path */ - bool hasSubstitute; - - /* Path info returned by the substituter's query info operation. */ - SubstitutablePathInfo info; - - /* Pipe for the substituter's standard output. */ - Pipe outPipe; - - /* Pipe for the substituter's standard error. */ - Pipe logPipe; - - /* The process ID of the builder. */ - Pid pid; - - /* Lock on the store path. */ - std::shared_ptr<PathLocks> outputLock; - - /* Whether to try to repair a valid path. */ - bool repair; - - /* Location where we're downloading the substitute. Differs from - storePath when doing a repair. */ - Path destPath; - - typedef void (SubstitutionGoal::*GoalState)(); - GoalState state; - -public: - SubstitutionGoal(const Path & storePath, Worker & worker, bool repair = false); - ~SubstitutionGoal(); - - void cancel(bool timeout); - - void work(); - - /* The states. */ - void init(); - void tryNext(); - void gotInfo(); - void referencesValid(); - void tryToRun(); - void finished(); - - /* Callback used by the worker to write to the log. */ - void handleChildOutput(int fd, const string & data); - void handleEOF(int fd); - - Path getStorePath() { return storePath; } -}; - - -SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker, bool repair) - : Goal(worker) - , hasSubstitute(false) - , repair(repair) -{ - this->storePath = storePath; - state = &SubstitutionGoal::init; - name = (format("substitution of `%1%'") % storePath).str(); - trace("created"); -} - - -SubstitutionGoal::~SubstitutionGoal() -{ - if (pid != -1) worker.childTerminated(pid); -} - - -void SubstitutionGoal::cancel(bool timeout) -{ - if (settings.printBuildTrace && timeout) - printMsg(lvlError, format("@ substituter-failed %1% timeout") % storePath); - if (pid != -1) { - pid_t savedPid = pid; - pid.kill(); - worker.childTerminated(savedPid); - } - amDone(ecFailed); -} - - -void SubstitutionGoal::work() -{ - (this->*state)(); -} - - -void SubstitutionGoal::init() -{ - trace("init"); - - worker.store.addTempRoot(storePath); - - /* If the path already exists we're done. */ - if (!repair && worker.store.isValidPath(storePath)) { - amDone(ecSuccess); - return; - } - - if (settings.readOnlyMode) - throw Error(format("cannot substitute path `%1%' - no write access to the Nix store") % storePath); - - subs = settings.substituters; - - tryNext(); -} - - -void SubstitutionGoal::tryNext() -{ - trace("trying next substituter"); - - if (subs.size() == 0) { - /* None left. Terminate this goal and let someone else deal - with it. */ - debug(format("path `%1%' is required, but there is no substituter that can build it") % storePath); - /* Hack: don't indicate failure if there were no substituters. - In that case the calling derivation should just do a - build. */ - amDone(hasSubstitute ? ecFailed : ecNoSubstituters); - return; - } - - sub = subs.front(); - subs.pop_front(); - - SubstitutablePathInfos infos; - PathSet dummy(singleton<PathSet>(storePath)); - worker.store.querySubstitutablePathInfos(sub, dummy, infos); - SubstitutablePathInfos::iterator k = infos.find(storePath); - if (k == infos.end()) { tryNext(); return; } - info = k->second; - hasSubstitute = true; - - /* To maintain the closure invariant, we first have to realise the - paths referenced by this one. */ - foreach (PathSet::iterator, i, info.references) - if (*i != storePath) /* ignore self-references */ - addWaitee(worker.makeSubstitutionGoal(*i)); - - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - referencesValid(); - else - state = &SubstitutionGoal::referencesValid; -} - - -void SubstitutionGoal::referencesValid() -{ - trace("all references realised"); - - if (nrFailed > 0) { - debug(format("some references of path `%1%' could not be realised") % storePath); - amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed); - return; - } - - foreach (PathSet::iterator, i, info.references) - if (*i != storePath) /* ignore self-references */ - assert(worker.store.isValidPath(*i)); - - state = &SubstitutionGoal::tryToRun; - worker.wakeUp(shared_from_this()); -} - - -void SubstitutionGoal::tryToRun() -{ - trace("trying to run"); - - /* Make sure that we are allowed to start a build. Note that even - is maxBuildJobs == 0 (no local builds allowed), we still allow - a substituter to run. This is because substitutions cannot be - distributed to another machine via the build hook. */ - if (worker.getNrLocalBuilds() >= (settings.maxBuildJobs == 0 ? 1 : settings.maxBuildJobs)) { - worker.waitForBuildSlot(shared_from_this()); - return; - } - - /* Maybe a derivation goal has already locked this path - (exceedingly unlikely, since it should have used a substitute - first, but let's be defensive). */ - outputLock.reset(); // make sure this goal's lock is gone - if (pathIsLockedByMe(storePath)) { - debug(format("restarting substitution of `%1%' because it's locked by another goal") - % storePath); - worker.waitForAnyGoal(shared_from_this()); - return; /* restart in the tryToRun() state when another goal finishes */ - } - - /* Acquire a lock on the output path. */ - outputLock = std::shared_ptr<PathLocks>(new PathLocks); - if (!outputLock->lockPaths(singleton<PathSet>(storePath), "", false)) { - worker.waitForAWhile(shared_from_this()); - return; - } - - /* Check again whether the path is invalid. */ - if (!repair && worker.store.isValidPath(storePath)) { - debug(format("store path `%1%' has become valid") % storePath); - outputLock->setDeletion(true); - amDone(ecSuccess); - return; - } - - printMsg(lvlInfo, format("fetching path `%1%'...") % storePath); - - outPipe.create(); - logPipe.create(); - - destPath = repair ? storePath + ".tmp" : storePath; - - /* Remove the (stale) output path if it exists. */ - if (pathExists(destPath)) - deletePath(destPath); - - worker.store.setSubstituterEnv(); - - /* Fill in the arguments. */ - Strings args; - args.push_back(baseNameOf(sub)); - args.push_back("--substitute"); - args.push_back(storePath); - args.push_back(destPath); - const char * * argArr = strings2CharPtrs(args); - - /* Fork the substitute program. */ - pid = maybeVfork(); - - switch (pid) { - - case -1: - throw SysError("unable to fork"); - - case 0: - try { /* child */ - - commonChildInit(logPipe); - - if (dup2(outPipe.writeSide, STDOUT_FILENO) == -1) - throw SysError("cannot dup output pipe into stdout"); - - execv(sub.c_str(), (char * *) argArr); - - throw SysError(format("executing `%1%'") % sub); - - } catch (std::exception & e) { - writeToStderr("substitute error: " + string(e.what()) + "\n"); - } - _exit(1); - } - - /* parent */ - pid.setSeparatePG(true); - pid.setKillSignal(SIGTERM); - outPipe.writeSide.close(); - logPipe.writeSide.close(); - worker.childStarted(shared_from_this(), - pid, singleton<set<int> >(logPipe.readSide), true, true); - - state = &SubstitutionGoal::finished; - - if (settings.printBuildTrace) - printMsg(lvlError, format("@ substituter-started %1% %2%") % storePath % sub); -} - - -void SubstitutionGoal::finished() -{ - trace("substitute finished"); - - /* Since we got an EOF on the logger pipe, the substitute is - presumed to have terminated. */ - pid_t savedPid = pid; - int status = pid.wait(true); - - /* So the child is gone now. */ - worker.childTerminated(savedPid); - - /* Close the read side of the logger pipe. */ - logPipe.readSide.close(); - - /* Get the hash info from stdout. */ - string dummy = readLine(outPipe.readSide); - string expectedHashStr = statusOk(status) ? readLine(outPipe.readSide) : ""; - outPipe.readSide.close(); - - /* Check the exit status and the build result. */ - HashResult hash; - try { - - if (!statusOk(status)) - throw SubstError(format("fetching path `%1%' %2%") - % storePath % statusToString(status)); - - if (!pathExists(destPath)) - throw SubstError(format("substitute did not produce path `%1%'") % destPath); - - hash = hashPath(htSHA256, destPath); - - /* Verify the expected hash we got from the substituer. */ - if (expectedHashStr != "") { - size_t n = expectedHashStr.find(':'); - if (n == string::npos) - throw Error(format("bad hash from substituter: %1%") % expectedHashStr); - HashType hashType = parseHashType(string(expectedHashStr, 0, n)); - if (hashType == htUnknown) - throw Error(format("unknown hash algorithm in `%1%'") % expectedHashStr); - Hash expectedHash = parseHash16or32(hashType, string(expectedHashStr, n + 1)); - Hash actualHash = hashType == htSHA256 ? hash.first : hashPath(hashType, destPath).first; - if (expectedHash != actualHash) - throw SubstError(format("hash mismatch in downloaded path `%1%': expected %2%, got %3%") - % storePath % printHash(expectedHash) % printHash(actualHash)); - } - - } catch (SubstError & e) { - - printMsg(lvlInfo, e.msg()); - - if (settings.printBuildTrace) { - printMsg(lvlError, format("@ substituter-failed %1% %2% %3%") - % storePath % status % e.msg()); - } - - /* Try the next substitute. */ - state = &SubstitutionGoal::tryNext; - worker.wakeUp(shared_from_this()); - return; - } - - if (repair) replaceValidPath(storePath, destPath); - - canonicalisePathMetaData(storePath, -1); - - worker.store.optimisePath(storePath); // FIXME: combine with hashPath() - - ValidPathInfo info2; - info2.path = storePath; - info2.hash = hash.first; - info2.narSize = hash.second; - info2.references = info.references; - info2.deriver = info.deriver; - worker.store.registerValidPath(info2); - - outputLock->setDeletion(true); - outputLock.reset(); - - worker.store.markContentsGood(storePath); - - printMsg(lvlChatty, - format("substitution of path `%1%' succeeded") % storePath); - - if (settings.printBuildTrace) - printMsg(lvlError, format("@ substituter-succeeded %1%") % storePath); - - amDone(ecSuccess); -} - - -void SubstitutionGoal::handleChildOutput(int fd, const string & data) -{ - assert(fd == logPipe.readSide); - if (verbosity >= settings.buildVerbosity) writeToStderr(data); - /* Don't write substitution output to a log file for now. We - probably should, though. */ -} - - -void SubstitutionGoal::handleEOF(int fd) -{ - if (fd == logPipe.readSide) worker.wakeUp(shared_from_this()); -} - - - -////////////////////////////////////////////////////////////////////// - - -static bool working = false; - - -Worker::Worker(LocalStore & store) - : store(store) -{ - /* Debugging: prevent recursive workers. */ - if (working) abort(); - working = true; - nrLocalBuilds = 0; - lastWokenUp = 0; - permanentFailure = false; -} - - -Worker::~Worker() -{ - working = false; - - /* Explicitly get rid of all strong pointers now. After this all - goals that refer to this worker should be gone. (Otherwise we - are in trouble, since goals may call childTerminated() etc. in - their destructors). */ - topGoals.clear(); -} - - -GoalPtr Worker::makeDerivationGoal(const Path & path, const StringSet & wantedOutputs, BuildMode buildMode) -{ - GoalPtr goal = derivationGoals[path].lock(); - if (!goal) { - goal = GoalPtr(new DerivationGoal(path, wantedOutputs, *this, buildMode)); - derivationGoals[path] = goal; - wakeUp(goal); - } else - (dynamic_cast<DerivationGoal *>(goal.get()))->addWantedOutputs(wantedOutputs); - return goal; -} - - -GoalPtr Worker::makeSubstitutionGoal(const Path & path, bool repair) -{ - GoalPtr goal = substitutionGoals[path].lock(); - if (!goal) { - goal = GoalPtr(new SubstitutionGoal(path, *this, repair)); - substitutionGoals[path] = goal; - wakeUp(goal); - } - return goal; -} - - -static void removeGoal(GoalPtr goal, WeakGoalMap & goalMap) -{ - /* !!! inefficient */ - for (WeakGoalMap::iterator i = goalMap.begin(); - i != goalMap.end(); ) - if (i->second.lock() == goal) { - WeakGoalMap::iterator j = i; ++j; - goalMap.erase(i); - i = j; - } - else ++i; -} - - -void Worker::removeGoal(GoalPtr goal) -{ - nix::removeGoal(goal, derivationGoals); - nix::removeGoal(goal, substitutionGoals); - if (topGoals.find(goal) != topGoals.end()) { - topGoals.erase(goal); - /* If a top-level goal failed, then kill all other goals - (unless keepGoing was set). */ - if (goal->getExitCode() == Goal::ecFailed && !settings.keepGoing) - topGoals.clear(); - } - - /* Wake up goals waiting for any goal to finish. */ - foreach (WeakGoals::iterator, i, waitingForAnyGoal) { - GoalPtr goal = i->lock(); - if (goal) wakeUp(goal); - } - - waitingForAnyGoal.clear(); -} - - -void Worker::wakeUp(GoalPtr goal) -{ - goal->trace("woken up"); - addToWeakGoals(awake, goal); -} - - -unsigned Worker::getNrLocalBuilds() -{ - return nrLocalBuilds; -} - - -void Worker::childStarted(GoalPtr goal, - pid_t pid, const set<int> & fds, bool inBuildSlot, - bool respectTimeouts) -{ - Child child; - child.goal = goal; - child.fds = fds; - child.timeStarted = child.lastOutput = time(0); - child.inBuildSlot = inBuildSlot; - child.respectTimeouts = respectTimeouts; - children[pid] = child; - if (inBuildSlot) nrLocalBuilds++; -} - - -void Worker::childTerminated(pid_t pid, bool wakeSleepers) -{ - assert(pid != -1); /* common mistake */ - - Children::iterator i = children.find(pid); - assert(i != children.end()); - - if (i->second.inBuildSlot) { - assert(nrLocalBuilds > 0); - nrLocalBuilds--; - } - - children.erase(pid); - - if (wakeSleepers) { - - /* Wake up goals waiting for a build slot. */ - foreach (WeakGoals::iterator, i, wantingToBuild) { - GoalPtr goal = i->lock(); - if (goal) wakeUp(goal); - } - - wantingToBuild.clear(); - } -} - - -void Worker::waitForBuildSlot(GoalPtr goal) -{ - debug("wait for build slot"); - if (getNrLocalBuilds() < settings.maxBuildJobs) - wakeUp(goal); /* we can do it right away */ - else - addToWeakGoals(wantingToBuild, goal); -} - - -void Worker::waitForAnyGoal(GoalPtr goal) -{ - debug("wait for any goal"); - addToWeakGoals(waitingForAnyGoal, goal); -} - - -void Worker::waitForAWhile(GoalPtr goal) -{ - debug("wait for a while"); - addToWeakGoals(waitingForAWhile, goal); -} - - -void Worker::run(const Goals & _topGoals) -{ - foreach (Goals::iterator, i, _topGoals) topGoals.insert(*i); - - startNest(nest, lvlDebug, format("entered goal loop")); - - while (1) { - - checkInterrupt(); - - /* Call every wake goal. */ - while (!awake.empty() && !topGoals.empty()) { - WeakGoals awake2(awake); - awake.clear(); - foreach (WeakGoals::iterator, i, awake2) { - checkInterrupt(); - GoalPtr goal = i->lock(); - if (goal) goal->work(); - if (topGoals.empty()) break; - } - } - - if (topGoals.empty()) break; - - /* Wait for input. */ - if (!children.empty() || !waitingForAWhile.empty()) - waitForInput(); - else { - if (awake.empty() && settings.maxBuildJobs == 0) throw Error( - "unable to start any build; either increase `--max-jobs' " - "or enable distributed builds"); - assert(!awake.empty()); - } - } - - /* If --keep-going is not set, it's possible that the main goal - exited while some of its subgoals were still active. But if - --keep-going *is* set, then they must all be finished now. */ - assert(!settings.keepGoing || awake.empty()); - assert(!settings.keepGoing || wantingToBuild.empty()); - assert(!settings.keepGoing || children.empty()); -} - - -void Worker::waitForInput() -{ - printMsg(lvlVomit, "waiting for children"); - - /* Process output from the file descriptors attached to the - children, namely log output and output path creation commands. - We also use this to detect child termination: if we get EOF on - the logger pipe of a build, we assume that the builder has - terminated. */ - - bool useTimeout = false; - struct timeval timeout; - timeout.tv_usec = 0; - time_t before = time(0); - - /* If we're monitoring for silence on stdout/stderr, or if there - is a build timeout, then wait for input until the first - deadline for any child. */ - assert(sizeof(time_t) >= sizeof(long)); - time_t nearest = LONG_MAX; // nearest deadline - foreach (Children::iterator, i, children) { - if (!i->second.respectTimeouts) continue; - if (settings.maxSilentTime != 0) - nearest = std::min(nearest, i->second.lastOutput + settings.maxSilentTime); - if (settings.buildTimeout != 0) - nearest = std::min(nearest, i->second.timeStarted + settings.buildTimeout); - } - if (nearest != LONG_MAX) { - timeout.tv_sec = std::max((time_t) 1, nearest - before); - useTimeout = true; - printMsg(lvlVomit, format("sleeping %1% seconds") % timeout.tv_sec); - } - - /* If we are polling goals that are waiting for a lock, then wake - up after a few seconds at most. */ - if (!waitingForAWhile.empty()) { - useTimeout = true; - if (lastWokenUp == 0) - printMsg(lvlError, "waiting for locks or build slots..."); - if (lastWokenUp == 0 || lastWokenUp > before) lastWokenUp = before; - timeout.tv_sec = std::max((time_t) 1, (time_t) (lastWokenUp + settings.pollInterval - before)); - } else lastWokenUp = 0; - - using namespace std; - /* Use select() to wait for the input side of any logger pipe to - become `available'. Note that `available' (i.e., non-blocking) - includes EOF. */ - fd_set fds; - FD_ZERO(&fds); - int fdMax = 0; - foreach (Children::iterator, i, children) { - foreach (set<int>::iterator, j, i->second.fds) { - FD_SET(*j, &fds); - if (*j >= fdMax) fdMax = *j + 1; - } - } - - if (select(fdMax, &fds, 0, 0, useTimeout ? &timeout : 0) == -1) { - if (errno == EINTR) return; - throw SysError("waiting for input"); - } - - time_t after = time(0); - - /* Process all available file descriptors. */ - - /* Since goals may be canceled from inside the loop below (causing - them go be erased from the `children' map), we have to be - careful that we don't keep iterators alive across calls to - cancel(). */ - set<pid_t> pids; - foreach (Children::iterator, i, children) pids.insert(i->first); - - foreach (set<pid_t>::iterator, i, pids) { - checkInterrupt(); - Children::iterator j = children.find(*i); - if (j == children.end()) continue; // child destroyed - GoalPtr goal = j->second.goal.lock(); - assert(goal); - - set<int> fds2(j->second.fds); - foreach (set<int>::iterator, k, fds2) { - if (FD_ISSET(*k, &fds)) { - unsigned char buffer[4096]; - ssize_t rd = read(*k, buffer, sizeof(buffer)); - if (rd == -1) { - if (errno != EINTR) - throw SysError(format("reading from %1%") - % goal->getName()); - } else if (rd == 0) { - debug(format("%1%: got EOF") % goal->getName()); - goal->handleEOF(*k); - j->second.fds.erase(*k); - } else { - printMsg(lvlVomit, format("%1%: read %2% bytes") - % goal->getName() % rd); - string data((char *) buffer, rd); - j->second.lastOutput = after; - goal->handleChildOutput(*k, data); - } - } - } - - if (goal->getExitCode() == Goal::ecBusy && - settings.maxSilentTime != 0 && - j->second.respectTimeouts && - after - j->second.lastOutput >= (time_t) settings.maxSilentTime) - { - printMsg(lvlError, - format("%1% timed out after %2% seconds of silence") - % goal->getName() % settings.maxSilentTime); - goal->cancel(true); - } - - else if (goal->getExitCode() == Goal::ecBusy && - settings.buildTimeout != 0 && - j->second.respectTimeouts && - after - j->second.timeStarted >= (time_t) settings.buildTimeout) - { - printMsg(lvlError, - format("%1% timed out after %2% seconds") - % goal->getName() % settings.buildTimeout); - goal->cancel(true); - } - } - - if (!waitingForAWhile.empty() && lastWokenUp + settings.pollInterval <= after) { - lastWokenUp = after; - foreach (WeakGoals::iterator, i, waitingForAWhile) { - GoalPtr goal = i->lock(); - if (goal) wakeUp(goal); - } - waitingForAWhile.clear(); - } -} - - -unsigned int Worker::exitStatus() -{ - return permanentFailure ? 100 : 1; -} - - -////////////////////////////////////////////////////////////////////// - - -void LocalStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode) -{ - startNest(nest, lvlDebug, - format("building %1%") % showPaths(drvPaths)); - - Worker worker(*this); - - Goals goals; - foreach (PathSet::const_iterator, i, drvPaths) { - DrvPathWithOutputs i2 = parseDrvPathWithOutputs(*i); - if (isDerivation(i2.first)) - goals.insert(worker.makeDerivationGoal(i2.first, i2.second, buildMode)); - else - goals.insert(worker.makeSubstitutionGoal(*i, buildMode)); - } - - worker.run(goals); - - PathSet failed; - foreach (Goals::iterator, i, goals) - if ((*i)->getExitCode() == Goal::ecFailed) { - DerivationGoal * i2 = dynamic_cast<DerivationGoal *>(i->get()); - if (i2) failed.insert(i2->getDrvPath()); - else failed.insert(dynamic_cast<SubstitutionGoal *>(i->get())->getStorePath()); - } - - if (!failed.empty()) - throw Error(format("build of %1% failed") % showPaths(failed), worker.exitStatus()); -} - - -void LocalStore::ensurePath(const Path & path) -{ - /* If the path is already valid, we're done. */ - if (isValidPath(path)) return; - - Worker worker(*this); - GoalPtr goal = worker.makeSubstitutionGoal(path); - Goals goals = singleton<Goals>(goal); - - worker.run(goals); - - if (goal->getExitCode() != Goal::ecSuccess) - throw Error(format("path `%1%' does not exist and cannot be created") % path, worker.exitStatus()); -} - - -void LocalStore::repairPath(const Path & path) -{ - Worker worker(*this); - GoalPtr goal = worker.makeSubstitutionGoal(path, true); - Goals goals = singleton<Goals>(goal); - - worker.run(goals); - - if (goal->getExitCode() != Goal::ecSuccess) - throw Error(format("cannot repair path `%1%'") % path, worker.exitStatus()); -} - - -} diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc deleted file mode 100644 index d91e42784c..0000000000 --- a/src/libstore/derivations.cc +++ /dev/null @@ -1,278 +0,0 @@ -#include "derivations.hh" -#include "store-api.hh" -#include "globals.hh" -#include "util.hh" -#include "misc.hh" - - -namespace nix { - - -void DerivationOutput::parseHashInfo(bool & recursive, HashType & hashType, Hash & hash) const -{ - recursive = false; - string algo = hashAlgo; - - if (string(algo, 0, 2) == "r:") { - recursive = true; - algo = string(algo, 2); - } - - hashType = parseHashType(algo); - if (hashType == htUnknown) - throw Error(format("unknown hash algorithm `%1%'") % algo); - - hash = parseHash(hashType, this->hash); -} - - -Path writeDerivation(StoreAPI & store, - const Derivation & drv, const string & name, bool repair) -{ - PathSet references; - references.insert(drv.inputSrcs.begin(), drv.inputSrcs.end()); - foreach (DerivationInputs::const_iterator, i, drv.inputDrvs) - references.insert(i->first); - /* Note that the outputs of a derivation are *not* references - (that can be missing (of course) and should not necessarily be - held during a garbage collection). */ - string suffix = name + drvExtension; - string contents = unparseDerivation(drv); - return settings.readOnlyMode - ? computeStorePathForText(suffix, contents, references) - : store.addTextToStore(suffix, contents, references, repair); -} - - -static Path parsePath(std::istream & str) -{ - string s = parseString(str); - if (s.size() == 0 || s[0] != '/') - throw Error(format("bad path `%1%' in derivation") % s); - return s; -} - - -static StringSet parseStrings(std::istream & str, bool arePaths) -{ - StringSet res; - while (!endOfList(str)) - res.insert(arePaths ? parsePath(str) : parseString(str)); - return res; -} - - -Derivation parseDerivation(const string & s) -{ - Derivation drv; - std::istringstream str(s); - expect(str, "Derive(["); - - /* Parse the list of outputs. */ - while (!endOfList(str)) { - DerivationOutput out; - expect(str, "("); string id = parseString(str); - expect(str, ","); out.path = parsePath(str); - expect(str, ","); out.hashAlgo = parseString(str); - expect(str, ","); out.hash = parseString(str); - expect(str, ")"); - drv.outputs[id] = out; - } - - /* Parse the list of input derivations. */ - expect(str, ",["); - while (!endOfList(str)) { - expect(str, "("); - Path drvPath = parsePath(str); - expect(str, ",["); - drv.inputDrvs[drvPath] = parseStrings(str, false); - expect(str, ")"); - } - - expect(str, ",["); drv.inputSrcs = parseStrings(str, true); - expect(str, ","); drv.platform = parseString(str); - expect(str, ","); drv.builder = parseString(str); - - /* Parse the builder arguments. */ - expect(str, ",["); - while (!endOfList(str)) - drv.args.push_back(parseString(str)); - - /* Parse the environment variables. */ - expect(str, ",["); - while (!endOfList(str)) { - expect(str, "("); string name = parseString(str); - expect(str, ","); string value = parseString(str); - expect(str, ")"); - drv.env[name] = value; - } - - expect(str, ")"); - return drv; -} - - -static void printString(string & res, const string & s) -{ - res += '"'; - for (const char * i = s.c_str(); *i; i++) - if (*i == '\"' || *i == '\\') { res += "\\"; res += *i; } - else if (*i == '\n') res += "\\n"; - else if (*i == '\r') res += "\\r"; - else if (*i == '\t') res += "\\t"; - else res += *i; - res += '"'; -} - - -template<class ForwardIterator> -static void printStrings(string & res, ForwardIterator i, ForwardIterator j) -{ - res += '['; - bool first = true; - for ( ; i != j; ++i) { - if (first) first = false; else res += ','; - printString(res, *i); - } - res += ']'; -} - - -string unparseDerivation(const Derivation & drv) -{ - string s; - s.reserve(65536); - s += "Derive(["; - - bool first = true; - foreach (DerivationOutputs::const_iterator, i, drv.outputs) { - if (first) first = false; else s += ','; - s += '('; printString(s, i->first); - s += ','; printString(s, i->second.path); - s += ','; printString(s, i->second.hashAlgo); - s += ','; printString(s, i->second.hash); - s += ')'; - } - - s += "],["; - first = true; - foreach (DerivationInputs::const_iterator, i, drv.inputDrvs) { - if (first) first = false; else s += ','; - s += '('; printString(s, i->first); - s += ','; printStrings(s, i->second.begin(), i->second.end()); - s += ')'; - } - - s += "],"; - printStrings(s, drv.inputSrcs.begin(), drv.inputSrcs.end()); - - s += ','; printString(s, drv.platform); - s += ','; printString(s, drv.builder); - s += ','; printStrings(s, drv.args.begin(), drv.args.end()); - - s += ",["; - first = true; - foreach (StringPairs::const_iterator, i, drv.env) { - if (first) first = false; else s += ','; - s += '('; printString(s, i->first); - s += ','; printString(s, i->second); - s += ')'; - } - - s += "])"; - - return s; -} - - -bool isDerivation(const string & fileName) -{ - return hasSuffix(fileName, drvExtension); -} - - -bool isFixedOutputDrv(const Derivation & drv) -{ - return drv.outputs.size() == 1 && - drv.outputs.begin()->first == "out" && - drv.outputs.begin()->second.hash != ""; -} - - -DrvHashes drvHashes; - - -/* Returns the hash of a derivation modulo fixed-output - subderivations. A fixed-output derivation is a derivation with one - output (`out') for which an expected hash and hash algorithm are - specified (using the `outputHash' and `outputHashAlgo' - attributes). We don't want changes to such derivations to - propagate upwards through the dependency graph, changing output - paths everywhere. - - For instance, if we change the url in a call to the `fetchurl' - function, we do not want to rebuild everything depending on it - (after all, (the hash of) the file being downloaded is unchanged). - So the *output paths* should not change. On the other hand, the - *derivation paths* should change to reflect the new dependency - graph. - - That's what this function does: it returns a hash which is just the - hash of the derivation ATerm, except that any input derivation - paths have been replaced by the result of a recursive call to this - function, and that for fixed-output derivations we return a hash of - its output path. */ -Hash hashDerivationModulo(StoreAPI & store, Derivation drv) -{ - /* Return a fixed hash for fixed-output derivations. */ - if (isFixedOutputDrv(drv)) { - DerivationOutputs::const_iterator i = drv.outputs.begin(); - return hashString(htSHA256, "fixed:out:" - + i->second.hashAlgo + ":" - + i->second.hash + ":" - + i->second.path); - } - - /* For other derivations, replace the inputs paths with recursive - calls to this function.*/ - DerivationInputs inputs2; - foreach (DerivationInputs::const_iterator, i, drv.inputDrvs) { - Hash h = drvHashes[i->first]; - if (h.type == htUnknown) { - assert(store.isValidPath(i->first)); - Derivation drv2 = parseDerivation(readFile(i->first)); - h = hashDerivationModulo(store, drv2); - drvHashes[i->first] = h; - } - inputs2[printHash(h)] = i->second; - } - drv.inputDrvs = inputs2; - - return hashString(htSHA256, unparseDerivation(drv)); -} - - -DrvPathWithOutputs parseDrvPathWithOutputs(const string & s) -{ - size_t n = s.find("!"); - return n == s.npos - ? DrvPathWithOutputs(s, std::set<string>()) - : DrvPathWithOutputs(string(s, 0, n), tokenizeString<std::set<string> >(string(s, n + 1), ",")); -} - - -Path makeDrvPathWithOutputs(const Path & drvPath, const std::set<string> & outputs) -{ - return outputs.empty() - ? drvPath - : drvPath + "!" + concatStringsSep(",", outputs); -} - - -bool wantOutput(const string & output, const std::set<string> & wanted) -{ - return wanted.empty() || wanted.find(output) != wanted.end(); -} - - -} diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh deleted file mode 100644 index 703410b925..0000000000 --- a/src/libstore/derivations.hh +++ /dev/null @@ -1,93 +0,0 @@ -#pragma once - -#include "types.hh" -#include "hash.hh" - -#include <map> - - -namespace nix { - - -/* Extension of derivations in the Nix store. */ -const string drvExtension = ".drv"; - - -/* Abstract syntax of derivations. */ - -struct DerivationOutput -{ - Path path; - string hashAlgo; /* hash used for expected hash computation */ - string hash; /* expected hash, may be null */ - DerivationOutput() - { - } - DerivationOutput(Path path, string hashAlgo, string hash) - { - this->path = path; - this->hashAlgo = hashAlgo; - this->hash = hash; - } - void parseHashInfo(bool & recursive, HashType & hashType, Hash & hash) const; -}; - -typedef std::map<string, DerivationOutput> DerivationOutputs; - -/* For inputs that are sub-derivations, we specify exactly which - output IDs we are interested in. */ -typedef std::map<Path, StringSet> DerivationInputs; - -typedef std::map<string, string> StringPairs; - -struct Derivation -{ - DerivationOutputs outputs; /* keyed on symbolic IDs */ - DerivationInputs inputDrvs; /* inputs that are sub-derivations */ - PathSet inputSrcs; /* inputs that are sources */ - string platform; - Path builder; - Strings args; - StringPairs env; -}; - - -class StoreAPI; - - -/* Write a derivation to the Nix store, and return its path. */ -Path writeDerivation(StoreAPI & store, - const Derivation & drv, const string & name, bool repair = false); - -/* Parse a derivation. */ -Derivation parseDerivation(const string & s); - -/* Print a derivation. */ -string unparseDerivation(const Derivation & drv); - -/* Check whether a file name ends with the extensions for - derivations. */ -bool isDerivation(const string & fileName); - -/* Return true iff this is a fixed-output derivation. */ -bool isFixedOutputDrv(const Derivation & drv); - -Hash hashDerivationModulo(StoreAPI & store, Derivation drv); - -/* Memoisation of hashDerivationModulo(). */ -typedef std::map<Path, Hash> DrvHashes; - -extern DrvHashes drvHashes; - -/* Split a string specifying a derivation and a set of outputs - (/nix/store/hash-foo!out1,out2,...) into the derivation path and - the outputs. */ -typedef std::pair<string, std::set<string> > DrvPathWithOutputs; -DrvPathWithOutputs parseDrvPathWithOutputs(const string & s); - -Path makeDrvPathWithOutputs(const Path & drvPath, const std::set<string> & outputs); - -bool wantOutput(const string & output, const std::set<string> & wanted); - - -} diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc deleted file mode 100644 index 96e891f876..0000000000 --- a/src/libstore/gc.cc +++ /dev/null @@ -1,748 +0,0 @@ -#include "globals.hh" -#include "misc.hh" -#include "local-store.hh" - -#include <functional> -#include <queue> -#include <algorithm> - -#include <sys/types.h> -#include <sys/stat.h> -#include <errno.h> -#include <fcntl.h> -#include <unistd.h> - - -namespace nix { - - -static string gcLockName = "gc.lock"; -static string tempRootsDir = "temproots"; -static string gcRootsDir = "gcroots"; - - -/* Acquire the global GC lock. This is used to prevent new Nix - processes from starting after the temporary root files have been - read. To be precise: when they try to create a new temporary root - file, they will block until the garbage collector has finished / - yielded the GC lock. */ -int LocalStore::openGCLock(LockType lockType) -{ - Path fnGCLock = (format("%1%/%2%") - % settings.nixStateDir % gcLockName).str(); - - debug(format("acquiring global GC lock `%1%'") % fnGCLock); - - AutoCloseFD fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT, 0600); - if (fdGCLock == -1) - throw SysError(format("opening global GC lock `%1%'") % fnGCLock); - closeOnExec(fdGCLock); - - if (!lockFile(fdGCLock, lockType, false)) { - printMsg(lvlError, format("waiting for the big garbage collector lock...")); - lockFile(fdGCLock, lockType, true); - } - - /* !!! Restrict read permission on the GC root. Otherwise any - process that can open the file for reading can DoS the - collector. */ - - return fdGCLock.borrow(); -} - - -static void makeSymlink(const Path & link, const Path & target) -{ - /* Create directories up to `gcRoot'. */ - createDirs(dirOf(link)); - - /* Create the new symlink. */ - Path tempLink = (format("%1%.tmp-%2%-%3%") - % link % getpid() % rand()).str(); - createSymlink(target, tempLink); - - /* Atomically replace the old one. */ - if (rename(tempLink.c_str(), link.c_str()) == -1) - throw SysError(format("cannot rename `%1%' to `%2%'") - % tempLink % link); -} - - -void LocalStore::syncWithGC() -{ - AutoCloseFD fdGCLock = openGCLock(ltRead); -} - - -void LocalStore::addIndirectRoot(const Path & path) -{ - string hash = printHash32(hashString(htSHA1, path)); - Path realRoot = canonPath((format("%1%/%2%/auto/%3%") - % settings.nixStateDir % gcRootsDir % hash).str()); - makeSymlink(realRoot, path); -} - - -Path addPermRoot(StoreAPI & store, const Path & _storePath, - const Path & _gcRoot, bool indirect, bool allowOutsideRootsDir) -{ - Path storePath(canonPath(_storePath)); - Path gcRoot(canonPath(_gcRoot)); - assertStorePath(storePath); - - if (isInStore(gcRoot)) - throw Error(format( - "creating a garbage collector root (%1%) in the Nix store is forbidden " - "(are you running nix-build inside the store?)") % gcRoot); - - if (indirect) { - /* Don't clobber the the link if it already exists and doesn't - point to the Nix store. */ - if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot)))) - throw Error(format("cannot create symlink `%1%'; already exists") % gcRoot); - makeSymlink(gcRoot, storePath); - store.addIndirectRoot(gcRoot); - } - - else { - if (!allowOutsideRootsDir) { - Path rootsDir = canonPath((format("%1%/%2%") % settings.nixStateDir % gcRootsDir).str()); - - if (string(gcRoot, 0, rootsDir.size() + 1) != rootsDir + "/") - throw Error(format( - "path `%1%' is not a valid garbage collector root; " - "it's not in the directory `%2%'") - % gcRoot % rootsDir); - } - - makeSymlink(gcRoot, storePath); - } - - /* Check that the root can be found by the garbage collector. - !!! This can be very slow on machines that have many roots. - Instead of reading all the roots, it would be more efficient to - check if the root is in a directory in or linked from the - gcroots directory. */ - if (settings.checkRootReachability) { - Roots roots = store.findRoots(); - if (roots.find(gcRoot) == roots.end()) - printMsg(lvlError, - format( - "warning: `%1%' is not in a directory where the garbage collector looks for roots; " - "therefore, `%2%' might be removed by the garbage collector") - % gcRoot % storePath); - } - - /* Grab the global GC root, causing us to block while a GC is in - progress. This prevents the set of permanent roots from - increasing while a GC is in progress. */ - store.syncWithGC(); - - return gcRoot; -} - - -/* The file to which we write our temporary roots. */ -static Path fnTempRoots; -static AutoCloseFD fdTempRoots; - - -void LocalStore::addTempRoot(const Path & path) -{ - /* Create the temporary roots file for this process. */ - if (fdTempRoots == -1) { - - while (1) { - Path dir = (format("%1%/%2%") % settings.nixStateDir % tempRootsDir).str(); - createDirs(dir); - - fnTempRoots = (format("%1%/%2%") - % dir % getpid()).str(); - - AutoCloseFD fdGCLock = openGCLock(ltRead); - - if (pathExists(fnTempRoots)) - /* It *must* be stale, since there can be no two - processes with the same pid. */ - unlink(fnTempRoots.c_str()); - - fdTempRoots = openLockFile(fnTempRoots, true); - - fdGCLock.close(); - - debug(format("acquiring read lock on `%1%'") % fnTempRoots); - lockFile(fdTempRoots, ltRead, true); - - /* Check whether the garbage collector didn't get in our - way. */ - struct stat st; - if (fstat(fdTempRoots, &st) == -1) - throw SysError(format("statting `%1%'") % fnTempRoots); - if (st.st_size == 0) break; - - /* The garbage collector deleted this file before we could - get a lock. (It won't delete the file after we get a - lock.) Try again. */ - } - - } - - /* Upgrade the lock to a write lock. This will cause us to block - if the garbage collector is holding our lock. */ - debug(format("acquiring write lock on `%1%'") % fnTempRoots); - lockFile(fdTempRoots, ltWrite, true); - - string s = path + '\0'; - writeFull(fdTempRoots, (const unsigned char *) s.data(), s.size()); - - /* Downgrade to a read lock. */ - debug(format("downgrading to read lock on `%1%'") % fnTempRoots); - lockFile(fdTempRoots, ltRead, true); -} - - -void removeTempRoots() -{ - if (fdTempRoots != -1) { - fdTempRoots.close(); - unlink(fnTempRoots.c_str()); - } -} - - -/* Automatically clean up the temporary roots file when we exit. */ -struct RemoveTempRoots -{ - ~RemoveTempRoots() - { - removeTempRoots(); - } -}; - -static RemoveTempRoots autoRemoveTempRoots __attribute__((unused)); - - -typedef std::shared_ptr<AutoCloseFD> FDPtr; -typedef list<FDPtr> FDs; - - -static void readTempRoots(PathSet & tempRoots, FDs & fds) -{ - /* Read the `temproots' directory for per-process temporary root - files. */ - Strings tempRootFiles = readDirectory( - (format("%1%/%2%") % settings.nixStateDir % tempRootsDir).str()); - - foreach (Strings::iterator, i, tempRootFiles) { - Path path = (format("%1%/%2%/%3%") % settings.nixStateDir % tempRootsDir % *i).str(); - - debug(format("reading temporary root file `%1%'") % path); - FDPtr fd(new AutoCloseFD(open(path.c_str(), O_RDWR, 0666))); - if (*fd == -1) { - /* It's okay if the file has disappeared. */ - if (errno == ENOENT) continue; - throw SysError(format("opening temporary roots file `%1%'") % path); - } - - /* This should work, but doesn't, for some reason. */ - //FDPtr fd(new AutoCloseFD(openLockFile(path, false))); - //if (*fd == -1) continue; - - /* Try to acquire a write lock without blocking. This can - only succeed if the owning process has died. In that case - we don't care about its temporary roots. */ - if (lockFile(*fd, ltWrite, false)) { - printMsg(lvlError, format("removing stale temporary roots file `%1%'") % path); - unlink(path.c_str()); - writeFull(*fd, (const unsigned char *) "d", 1); - continue; - } - - /* Acquire a read lock. This will prevent the owning process - from upgrading to a write lock, therefore it will block in - addTempRoot(). */ - debug(format("waiting for read lock on `%1%'") % path); - lockFile(*fd, ltRead, true); - - /* Read the entire file. */ - string contents = readFile(*fd); - - /* Extract the roots. */ - string::size_type pos = 0, end; - - while ((end = contents.find((char) 0, pos)) != string::npos) { - Path root(contents, pos, end - pos); - debug(format("got temporary root `%1%'") % root); - assertStorePath(root); - tempRoots.insert(root); - pos = end + 1; - } - - fds.push_back(fd); /* keep open */ - } -} - - -static void foundRoot(StoreAPI & store, - const Path & path, const Path & target, Roots & roots) -{ - Path storePath = toStorePath(target); - if (store.isValidPath(storePath)) - roots[path] = storePath; - else - printMsg(lvlInfo, format("skipping invalid root from `%1%' to `%2%'") % path % storePath); -} - - -static void findRoots(StoreAPI & store, const Path & path, Roots & roots) -{ - try { - - struct stat st = lstat(path); - - if (S_ISDIR(st.st_mode)) { - Strings names = readDirectory(path); - foreach (Strings::iterator, i, names) - findRoots(store, path + "/" + *i, roots); - } - - else if (S_ISLNK(st.st_mode)) { - Path target = readLink(path); - if (isInStore(target)) - foundRoot(store, path, target, roots); - - /* Handle indirect roots. */ - else { - target = absPath(target, dirOf(path)); - if (!pathExists(target)) { - if (isInDir(path, settings.nixStateDir + "/" + gcRootsDir + "/auto")) { - printMsg(lvlInfo, format("removing stale link from `%1%' to `%2%'") % path % target); - unlink(path.c_str()); - } - } else { - struct stat st2 = lstat(target); - if (!S_ISLNK(st2.st_mode)) return; - Path target2 = readLink(target); - if (isInStore(target2)) foundRoot(store, target, target2, roots); - } - } - } - - } - - catch (SysError & e) { - /* We only ignore permanent failures. */ - if (e.errNo == EACCES || e.errNo == ENOENT || e.errNo == ENOTDIR) - printMsg(lvlInfo, format("cannot read potential root `%1%'") % path); - else - throw; - } -} - - -Roots LocalStore::findRoots() -{ - Roots roots; - - /* Process direct roots in {gcroots,manifests,profiles}. */ - nix::findRoots(*this, settings.nixStateDir + "/" + gcRootsDir, roots); - nix::findRoots(*this, settings.nixStateDir + "/manifests", roots); - nix::findRoots(*this, settings.nixStateDir + "/profiles", roots); - - return roots; -} - - -static void addAdditionalRoots(StoreAPI & store, PathSet & roots) -{ - Path rootFinder = getEnv("NIX_ROOT_FINDER", - settings.nixLibexecDir + "/nix/find-runtime-roots.pl"); - - if (rootFinder.empty()) return; - - debug(format("executing `%1%' to find additional roots") % rootFinder); - - string result = runProgram(rootFinder); - - StringSet paths = tokenizeString<StringSet>(result, "\n"); - - foreach (StringSet::iterator, i, paths) { - if (isInStore(*i)) { - Path path = toStorePath(*i); - if (roots.find(path) == roots.end() && store.isValidPath(path)) { - debug(format("got additional root `%1%'") % path); - roots.insert(path); - } - } - } -} - - -struct GCLimitReached { }; - - -struct LocalStore::GCState -{ - GCOptions options; - GCResults & results; - PathSet roots; - PathSet tempRoots; - PathSet dead; - PathSet alive; - bool gcKeepOutputs; - bool gcKeepDerivations; - unsigned long long bytesInvalidated; - Path trashDir; - bool shouldDelete; - GCState(GCResults & results_) : results(results_), bytesInvalidated(0) { } -}; - - -bool LocalStore::isActiveTempFile(const GCState & state, - const Path & path, const string & suffix) -{ - return hasSuffix(path, suffix) - && state.tempRoots.find(string(path, 0, path.size() - suffix.size())) != state.tempRoots.end(); -} - - -void LocalStore::deleteGarbage(GCState & state, const Path & path) -{ - unsigned long long bytesFreed; - deletePath(path, bytesFreed); - state.results.bytesFreed += bytesFreed; -} - - -void LocalStore::deletePathRecursive(GCState & state, const Path & path) -{ - checkInterrupt(); - - unsigned long long size = 0; - - if (isValidPath(path)) { - PathSet referrers; - queryReferrers(path, referrers); - foreach (PathSet::iterator, i, referrers) - if (*i != path) deletePathRecursive(state, *i); - size = queryPathInfo(path).narSize; - invalidatePathChecked(path); - } - - struct stat st; - if (lstat(path.c_str(), &st)) { - if (errno == ENOENT) return; - throw SysError(format("getting status of %1%") % path); - } - - printMsg(lvlInfo, format("deleting `%1%'") % path); - - state.results.paths.insert(path); - - /* If the path is not a regular file or symlink, move it to the - trash directory. The move is to ensure that later (when we're - not holding the global GC lock) we can delete the path without - being afraid that the path has become alive again. Otherwise - delete it right away. */ - if (S_ISDIR(st.st_mode)) { - // Estimate the amount freed using the narSize field. FIXME: - // if the path was not valid, need to determine the actual - // size. - state.bytesInvalidated += size; - // Mac OS X cannot rename directories if they are read-only. - if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1) - throw SysError(format("making `%1%' writable") % path); - Path tmp = state.trashDir + "/" + baseNameOf(path); - if (rename(path.c_str(), tmp.c_str())) - throw SysError(format("unable to rename `%1%' to `%2%'") % path % tmp); - } else - deleteGarbage(state, path); - - if (state.results.bytesFreed + state.bytesInvalidated > state.options.maxFreed) { - printMsg(lvlInfo, format("deleted or invalidated more than %1% bytes; stopping") % state.options.maxFreed); - throw GCLimitReached(); - } -} - - -bool LocalStore::canReachRoot(GCState & state, PathSet & visited, const Path & path) -{ - if (visited.find(path) != visited.end()) return false; - - if (state.alive.find(path) != state.alive.end()) { - return true; - } - - if (state.dead.find(path) != state.dead.end()) { - return false; - } - - if (state.roots.find(path) != state.roots.end()) { - printMsg(lvlDebug, format("cannot delete `%1%' because it's a root") % path); - state.alive.insert(path); - return true; - } - - visited.insert(path); - - if (!isValidPath(path)) return false; - - PathSet incoming; - - /* Don't delete this path if any of its referrers are alive. */ - queryReferrers(path, incoming); - - /* If gc-keep-derivations is set and this is a derivation, then - don't delete the derivation if any of the outputs are alive. */ - if (state.gcKeepDerivations && isDerivation(path)) { - PathSet outputs = queryDerivationOutputs(path); - foreach (PathSet::iterator, i, outputs) - if (isValidPath(*i) && queryDeriver(*i) == path) - incoming.insert(*i); - } - - /* If gc-keep-outputs is set, then don't delete this path if there - are derivers of this path that are not garbage. */ - if (state.gcKeepOutputs) { - PathSet derivers = queryValidDerivers(path); - foreach (PathSet::iterator, i, derivers) - incoming.insert(*i); - } - - foreach (PathSet::iterator, i, incoming) - if (*i != path) - if (canReachRoot(state, visited, *i)) { - state.alive.insert(path); - return true; - } - - return false; -} - - -void LocalStore::tryToDelete(GCState & state, const Path & path) -{ - checkInterrupt(); - - if (path == linksDir || path == state.trashDir) return; - - startNest(nest, lvlDebug, format("considering whether to delete `%1%'") % path); - - if (!isValidPath(path)) { - /* A lock file belonging to a path that we're building right - now isn't garbage. */ - if (isActiveTempFile(state, path, ".lock")) return; - - /* Don't delete .chroot directories for derivations that are - currently being built. */ - if (isActiveTempFile(state, path, ".chroot")) return; - } - - PathSet visited; - - if (canReachRoot(state, visited, path)) { - printMsg(lvlDebug, format("cannot delete `%1%' because it's still reachable") % path); - } else { - /* No path we visited was a root, so everything is garbage. - But we only delete âpathâ and its referrers here so that - ânix-store --deleteâ doesn't have the unexpected effect of - recursing into derivations and outputs. */ - state.dead.insert(visited.begin(), visited.end()); - if (state.shouldDelete) - deletePathRecursive(state, path); - } -} - - -/* Unlink all files in /nix/store/.links that have a link count of 1, - which indicates that there are no other links and so they can be - safely deleted. FIXME: race condition with optimisePath(): we - might see a link count of 1 just before optimisePath() increases - the link count. */ -void LocalStore::removeUnusedLinks(const GCState & state) -{ - AutoCloseDir dir = opendir(linksDir.c_str()); - if (!dir) throw SysError(format("opening directory `%1%'") % linksDir); - - long long actualSize = 0, unsharedSize = 0; - - struct dirent * dirent; - while (errno = 0, dirent = readdir(dir)) { - checkInterrupt(); - string name = dirent->d_name; - if (name == "." || name == "..") continue; - Path path = linksDir + "/" + name; - - struct stat st; - if (lstat(path.c_str(), &st) == -1) - throw SysError(format("statting `%1%'") % path); - - if (st.st_nlink != 1) { - unsigned long long size = st.st_blocks * 512ULL; - actualSize += size; - unsharedSize += (st.st_nlink - 1) * size; - continue; - } - - printMsg(lvlTalkative, format("deleting unused link `%1%'") % path); - - if (unlink(path.c_str()) == -1) - throw SysError(format("deleting `%1%'") % path); - - state.results.bytesFreed += st.st_blocks * 512; - } - - struct stat st; - if (stat(linksDir.c_str(), &st) == -1) - throw SysError(format("statting `%1%'") % linksDir); - long long overhead = st.st_blocks * 512ULL; - - printMsg(lvlInfo, format("note: currently hard linking saves %.2f MiB") - % ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0))); -} - - -void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) -{ - GCState state(results); - state.options = options; - state.trashDir = settings.nixStore + "/trash"; - state.gcKeepOutputs = settings.gcKeepOutputs; - state.gcKeepDerivations = settings.gcKeepDerivations; - - /* Using `--ignore-liveness' with `--delete' can have unintended - consequences if `gc-keep-outputs' or `gc-keep-derivations' are - true (the garbage collector will recurse into deleting the - outputs or derivers, respectively). So disable them. */ - if (options.action == GCOptions::gcDeleteSpecific && options.ignoreLiveness) { - state.gcKeepOutputs = false; - state.gcKeepDerivations = false; - } - - state.shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific; - - /* Acquire the global GC root. This prevents - a) New roots from being added. - b) Processes from creating new temporary root files. */ - AutoCloseFD fdGCLock = openGCLock(ltWrite); - - /* Find the roots. Since we've grabbed the GC lock, the set of - permanent roots cannot increase now. */ - printMsg(lvlError, format("finding garbage collector roots...")); - Roots rootMap = options.ignoreLiveness ? Roots() : findRoots(); - - foreach (Roots::iterator, i, rootMap) state.roots.insert(i->second); - - /* Add additional roots returned by the program specified by the - NIX_ROOT_FINDER environment variable. This is typically used - to add running programs to the set of roots (to prevent them - from being garbage collected). */ - if (!options.ignoreLiveness) - addAdditionalRoots(*this, state.roots); - - /* Read the temporary roots. This acquires read locks on all - per-process temporary root files. So after this point no paths - can be added to the set of temporary roots. */ - FDs fds; - readTempRoots(state.tempRoots, fds); - state.roots.insert(state.tempRoots.begin(), state.tempRoots.end()); - - /* After this point the set of roots or temporary roots cannot - increase, since we hold locks on everything. So everything - that is not reachable from `roots'. */ - - if (state.shouldDelete) { - if (pathExists(state.trashDir)) deleteGarbage(state, state.trashDir); - createDirs(state.trashDir); - } - - /* Now either delete all garbage paths, or just the specified - paths (for gcDeleteSpecific). */ - - if (options.action == GCOptions::gcDeleteSpecific) { - - foreach (PathSet::iterator, i, options.pathsToDelete) { - assertStorePath(*i); - tryToDelete(state, *i); - if (state.dead.find(*i) == state.dead.end()) - throw Error(format("cannot delete path `%1%' since it is still alive") % *i); - } - - } else if (options.maxFreed > 0) { - - if (state.shouldDelete) - printMsg(lvlError, format("deleting garbage...")); - else - printMsg(lvlError, format("determining live/dead paths...")); - - try { - - AutoCloseDir dir = opendir(settings.nixStore.c_str()); - if (!dir) throw SysError(format("opening directory `%1%'") % settings.nixStore); - - /* Read the store and immediately delete all paths that - aren't valid. When using --max-freed etc., deleting - invalid paths is preferred over deleting unreachable - paths, since unreachable paths could become reachable - again. We don't use readDirectory() here so that GCing - can start faster. */ - Paths entries; - struct dirent * dirent; - while (errno = 0, dirent = readdir(dir)) { - checkInterrupt(); - string name = dirent->d_name; - if (name == "." || name == "..") continue; - Path path = settings.nixStore + "/" + name; - if (isValidPath(path)) - entries.push_back(path); - else - tryToDelete(state, path); - } - - dir.close(); - - /* Now delete the unreachable valid paths. Randomise the - order in which we delete entries to make the collector - less biased towards deleting paths that come - alphabetically first (e.g. /nix/store/000...). This - matters when using --max-freed etc. */ - vector<Path> entries_(entries.begin(), entries.end()); - random_shuffle(entries_.begin(), entries_.end()); - - foreach (vector<Path>::iterator, i, entries_) - tryToDelete(state, *i); - - } catch (GCLimitReached & e) { - } - } - - if (state.options.action == GCOptions::gcReturnLive) { - state.results.paths = state.alive; - return; - } - - if (state.options.action == GCOptions::gcReturnDead) { - state.results.paths = state.dead; - return; - } - - /* Allow other processes to add to the store from here on. */ - fdGCLock.close(); - fds.clear(); - - /* Delete the trash directory. */ - printMsg(lvlInfo, format("deleting `%1%'") % state.trashDir); - deleteGarbage(state, state.trashDir); - - /* Clean up the links directory. */ - if (options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific) { - printMsg(lvlError, format("deleting unused links...")); - removeUnusedLinks(state); - } - - /* While we're at it, vacuum the database. */ - if (options.action == GCOptions::gcDeleteDead) vacuumDB(); -} - - -} diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc deleted file mode 100644 index 86fa56739c..0000000000 --- a/src/libstore/globals.cc +++ /dev/null @@ -1,240 +0,0 @@ -#include "config.h" - -#include "globals.hh" -#include "util.hh" - -#include <map> -#include <algorithm> - - -namespace nix { - - -/* The default location of the daemon socket, relative to nixStateDir. - The socket is in a directory to allow you to control access to the - Nix daemon by setting the mode/ownership of the directory - appropriately. (This wouldn't work on the socket itself since it - must be deleted and recreated on startup.) */ -#define DEFAULT_SOCKET_PATH "/daemon-socket/socket" - - -Settings settings; - - -Settings::Settings() -{ - keepFailed = false; - keepGoing = false; - tryFallback = false; - buildVerbosity = lvlError; - maxBuildJobs = 1; - buildCores = 1; - readOnlyMode = false; - thisSystem = SYSTEM; - maxSilentTime = 0; - buildTimeout = 0; - useBuildHook = true; - printBuildTrace = false; - reservedSize = 1024 * 1024; - fsyncMetadata = true; - useSQLiteWAL = true; - syncBeforeRegistering = false; - useSubstitutes = true; - useChroot = false; - useSshSubstituter = false; - impersonateLinux26 = false; - keepLog = true; - compressLog = true; - maxLogSize = 0; - cacheFailure = false; - pollInterval = 5; - checkRootReachability = false; - gcKeepOutputs = false; - gcKeepDerivations = true; - autoOptimiseStore = false; - envKeepDerivations = false; - lockCPU = getEnv("NIX_AFFINITY_HACK", "1") == "1"; - showTrace = false; -} - - -void Settings::processEnvironment() -{ - nixStore = canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR))); - nixDataDir = canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR)); - nixLogDir = canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR)); - nixStateDir = canonPath(getEnv("NIX_STATE_DIR", NIX_STATE_DIR)); - nixDBPath = getEnv("NIX_DB_DIR", nixStateDir + "/db"); - nixConfDir = canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR)); - nixLibexecDir = canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR)); - nixBinDir = canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR)); - nixDaemonSocketFile = canonPath(nixStateDir + DEFAULT_SOCKET_PATH); -} - - -void Settings::loadConfFile() -{ - Path settingsFile = (format("%1%/%2%") % nixConfDir % "nix.conf").str(); - if (!pathExists(settingsFile)) return; - string contents = readFile(settingsFile); - - unsigned int pos = 0; - - while (pos < contents.size()) { - string line; - while (pos < contents.size() && contents[pos] != '\n') - line += contents[pos++]; - pos++; - - string::size_type hash = line.find('#'); - if (hash != string::npos) - line = string(line, 0, hash); - - vector<string> tokens = tokenizeString<vector<string> >(line); - if (tokens.empty()) continue; - - if (tokens.size() < 2 || tokens[1] != "=") - throw Error(format("illegal configuration line `%1%' in `%2%'") % line % settingsFile); - - string name = tokens[0]; - - vector<string>::iterator i = tokens.begin(); - advance(i, 2); - settings[name] = concatStringsSep(" ", Strings(i, tokens.end())); // FIXME: slow - }; -} - - -void Settings::set(const string & name, const string & value) -{ - settings[name] = value; - overrides[name] = value; -} - - -void Settings::update() -{ - get(tryFallback, "build-fallback"); - get(maxBuildJobs, "build-max-jobs"); - get(buildCores, "build-cores"); - get(thisSystem, "system"); - get(maxSilentTime, "build-max-silent-time"); - get(buildTimeout, "build-timeout"); - get(reservedSize, "gc-reserved-space"); - get(fsyncMetadata, "fsync-metadata"); - get(useSQLiteWAL, "use-sqlite-wal"); - get(syncBeforeRegistering, "sync-before-registering"); - get(useSubstitutes, "build-use-substitutes"); - get(buildUsersGroup, "build-users-group"); - get(useChroot, "build-use-chroot"); - get(dirsInChroot, "build-chroot-dirs"); - get(impersonateLinux26, "build-impersonate-linux-26"); - get(keepLog, "build-keep-log"); - get(compressLog, "build-compress-log"); - get(maxLogSize, "build-max-log-size"); - get(cacheFailure, "build-cache-failure"); - get(pollInterval, "build-poll-interval"); - get(checkRootReachability, "gc-check-reachability"); - get(gcKeepOutputs, "gc-keep-outputs"); - get(gcKeepDerivations, "gc-keep-derivations"); - get(autoOptimiseStore, "auto-optimise-store"); - get(envKeepDerivations, "env-keep-derivations"); - get(sshSubstituterHosts, "ssh-substituter-hosts"); - get(useSshSubstituter, "use-ssh-substituter"); - - string subs = getEnv("NIX_SUBSTITUTERS", "default"); - if (subs == "default") { - substituters.clear(); -#if 0 - if (getEnv("NIX_OTHER_STORES") != "") - substituters.push_back(nixLibexecDir + "/nix/substituters/copy-from-other-stores.pl"); -#endif - substituters.push_back(nixLibexecDir + "/nix/substituters/download-using-manifests.pl"); - substituters.push_back(nixLibexecDir + "/nix/substituters/download-from-binary-cache.pl"); - if (useSshSubstituter) - substituters.push_back(nixLibexecDir + "/nix/substituters/download-via-ssh"); - } else - substituters = tokenizeString<Strings>(subs, ":"); -} - - -void Settings::get(string & res, const string & name) -{ - SettingsMap::iterator i = settings.find(name); - if (i == settings.end()) return; - res = i->second; -} - - -void Settings::get(bool & res, const string & name) -{ - SettingsMap::iterator i = settings.find(name); - if (i == settings.end()) return; - if (i->second == "true") res = true; - else if (i->second == "false") res = false; - else throw Error(format("configuration option `%1%' should be either `true' or `false', not `%2%'") - % name % i->second); -} - - -void Settings::get(StringSet & res, const string & name) -{ - SettingsMap::iterator i = settings.find(name); - if (i == settings.end()) return; - res.clear(); - Strings ss = tokenizeString<Strings>(i->second); - res.insert(ss.begin(), ss.end()); -} - -void Settings::get(Strings & res, const string & name) -{ - SettingsMap::iterator i = settings.find(name); - if (i == settings.end()) return; - res = tokenizeString<Strings>(i->second); -} - - -template<class N> void Settings::get(N & res, const string & name) -{ - SettingsMap::iterator i = settings.find(name); - if (i == settings.end()) return; - if (!string2Int(i->second, res)) - throw Error(format("configuration setting `%1%' should have an integer value") % name); -} - - -string Settings::pack() -{ - string s; - foreach (SettingsMap::iterator, i, settings) { - if (i->first.find('\n') != string::npos || - i->first.find('=') != string::npos || - i->second.find('\n') != string::npos) - throw Error("illegal option name/value"); - s += i->first; s += '='; s += i->second; s += '\n'; - } - return s; -} - - -void Settings::unpack(const string & pack) { - Strings lines = tokenizeString<Strings>(pack, "\n"); - foreach (Strings::iterator, i, lines) { - string::size_type eq = i->find('='); - if (eq == string::npos) - throw Error("illegal option name/value"); - set(i->substr(0, eq), i->substr(eq + 1)); - } -} - - -Settings::SettingsMap Settings::getOverrides() -{ - return overrides; -} - - -const string nixVersion = PACKAGE_VERSION; - - -} diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh deleted file mode 100644 index 711c365294..0000000000 --- a/src/libstore/globals.hh +++ /dev/null @@ -1,218 +0,0 @@ -#pragma once - -#include "types.hh" - -#include <map> -#include <sys/types.h> - - -namespace nix { - - -struct Settings { - - typedef std::map<string, string> SettingsMap; - - Settings(); - - void processEnvironment(); - - void loadConfFile(); - - void set(const string & name, const string & value); - - void update(); - - string pack(); - - void unpack(const string & pack); - - SettingsMap getOverrides(); - - /* The directory where we store sources and derived files. */ - Path nixStore; - - Path nixDataDir; /* !!! fix */ - - /* The directory where we log various operations. */ - Path nixLogDir; - - /* The directory where state is stored. */ - Path nixStateDir; - - /* The directory where we keep the SQLite database. */ - Path nixDBPath; - - /* The directory where configuration files are stored. */ - Path nixConfDir; - - /* The directory where internal helper programs are stored. */ - Path nixLibexecDir; - - /* The directory where the main programs are stored. */ - Path nixBinDir; - - /* File name of the socket the daemon listens to. */ - Path nixDaemonSocketFile; - - /* Whether to keep temporary directories of failed builds. */ - bool keepFailed; - - /* Whether to keep building subgoals when a sibling (another - subgoal of the same goal) fails. */ - bool keepGoing; - - /* Whether, if we cannot realise the known closure corresponding - to a derivation, we should try to normalise the derivation - instead. */ - bool tryFallback; - - /* Verbosity level for build output. */ - Verbosity buildVerbosity; - - /* Maximum number of parallel build jobs. 0 means unlimited. */ - unsigned int maxBuildJobs; - - /* Number of CPU cores to utilize in parallel within a build, - i.e. by passing this number to Make via '-j'. 0 means that the - number of actual CPU cores on the local host ought to be - auto-detected. */ - unsigned int buildCores; - - /* Read-only mode. Don't copy stuff to the store, don't change - the database. */ - bool readOnlyMode; - - /* The canonical system name, as returned by config.guess. */ - string thisSystem; - - /* The maximum time in seconds that a builer can go without - producing any output on stdout/stderr before it is killed. 0 - means infinity. */ - time_t maxSilentTime; - - /* The maximum duration in seconds that a builder can run. 0 - means infinity. */ - time_t buildTimeout; - - /* The substituters. There are programs that can somehow realise - a store path without building, e.g., by downloading it or - copying it from a CD. */ - Paths substituters; - - /* Whether to use build hooks (for distributed builds). Sometimes - users want to disable this from the command-line. */ - bool useBuildHook; - - /* Whether buildDerivations() should print out lines on stderr in - a fixed format to allow its progress to be monitored. Each - line starts with a "@". The following are defined: - - @ build-started <drvpath> <outpath> <system> <logfile> - @ build-failed <drvpath> <outpath> <exitcode> <error text> - @ build-succeeded <drvpath> <outpath> - @ substituter-started <outpath> <substituter> - @ substituter-failed <outpath> <exitcode> <error text> - @ substituter-succeeded <outpath> - - Best combined with --no-build-output, otherwise stderr might - conceivably contain lines in this format printed by the - builders. */ - bool printBuildTrace; - - /* Amount of reserved space for the garbage collector - (/nix/var/nix/db/reserved). */ - off_t reservedSize; - - /* Whether SQLite should use fsync. */ - bool fsyncMetadata; - - /* Whether SQLite should use WAL mode. */ - bool useSQLiteWAL; - - /* Whether to call sync() before registering a path as valid. */ - bool syncBeforeRegistering; - - /* Whether to use substitutes. */ - bool useSubstitutes; - - /* The Unix group that contains the build users. */ - string buildUsersGroup; - - /* Whether to build in chroot. */ - bool useChroot; - - /* The directories from the host filesystem to be included in the - chroot. */ - StringSet dirsInChroot; - - /* Set of ssh connection strings for the ssh substituter */ - Strings sshSubstituterHosts; - - /* Whether to use the ssh substituter at all */ - bool useSshSubstituter; - - /* Whether to impersonate a Linux 2.6 machine on newer kernels. */ - bool impersonateLinux26; - - /* Whether to store build logs. */ - bool keepLog; - - /* Whether to compress logs. */ - bool compressLog; - - /* Maximum number of bytes a builder can write to stdout/stderr - before being killed (0 means no limit). */ - unsigned long maxLogSize; - - /* Whether to cache build failures. */ - bool cacheFailure; - - /* How often (in seconds) to poll for locks. */ - unsigned int pollInterval; - - /* Whether to check if new GC roots can in fact be found by the - garbage collector. */ - bool checkRootReachability; - - /* Whether the garbage collector should keep outputs of live - derivations. */ - bool gcKeepOutputs; - - /* Whether the garbage collector should keep derivers of live - paths. */ - bool gcKeepDerivations; - - /* Whether to automatically replace files with identical contents - with hard links. */ - bool autoOptimiseStore; - - /* Whether to add derivations as a dependency of user environments - (to prevent them from being GCed). */ - bool envKeepDerivations; - - /* Whether to lock the Nix client and worker to the same CPU. */ - bool lockCPU; - - /* Whether to show a stack trace if Nix evaluation fails. */ - bool showTrace; - -private: - SettingsMap settings, overrides; - - void get(string & res, const string & name); - void get(bool & res, const string & name); - void get(StringSet & res, const string & name); - void get(Strings & res, const string & name); - template<class N> void get(N & res, const string & name); -}; - - -// FIXME: don't use a global variable. -extern Settings settings; - - -extern const string nixVersion; - - -} diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc deleted file mode 100644 index 1293a6e8f2..0000000000 --- a/src/libstore/local-store.cc +++ /dev/null @@ -1,2010 +0,0 @@ -#include "config.h" -#include "local-store.hh" -#include "globals.hh" -#include "archive.hh" -#include "pathlocks.hh" -#include "worker-protocol.hh" -#include "derivations.hh" -#include "affinity.hh" - -#include <iostream> -#include <algorithm> -#include <cstring> - -#include <sys/types.h> -#include <sys/stat.h> -#include <sys/time.h> -#include <unistd.h> -#include <utime.h> -#include <fcntl.h> -#include <errno.h> -#include <stdio.h> -#include <time.h> - -#if HAVE_UNSHARE && HAVE_STATVFS && HAVE_SYS_MOUNT_H -#include <sched.h> -#include <sys/statvfs.h> -#include <sys/mount.h> -#endif - -#if HAVE_LINUX_FS_H -#include <linux/fs.h> -#include <sys/ioctl.h> -#include <errno.h> -#endif - -#include <sqlite3.h> - - -namespace nix { - - -MakeError(SQLiteError, Error); -MakeError(SQLiteBusy, SQLiteError); - - -static void throwSQLiteError(sqlite3 * db, const format & f) - __attribute__ ((noreturn)); - -static void throwSQLiteError(sqlite3 * db, const format & f) -{ - int err = sqlite3_errcode(db); - if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) { - if (err == SQLITE_PROTOCOL) - printMsg(lvlError, "warning: SQLite database is busy (SQLITE_PROTOCOL)"); - else { - static bool warned = false; - if (!warned) { - printMsg(lvlError, "warning: SQLite database is busy"); - warned = true; - } - } - /* Sleep for a while since retrying the transaction right away - is likely to fail again. */ -#if HAVE_NANOSLEEP - struct timespec t; - t.tv_sec = 0; - t.tv_nsec = (random() % 100) * 1000 * 1000; /* <= 0.1s */ - nanosleep(&t, 0); -#else - sleep(1); -#endif - throw SQLiteBusy(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)); - } - else - throw SQLiteError(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)); -} - - -/* Convenience macros for retrying a SQLite transaction. */ -#define retry_sqlite while (1) { try { -#define end_retry_sqlite break; } catch (SQLiteBusy & e) { } } - - -SQLite::~SQLite() -{ - try { - if (db && sqlite3_close(db) != SQLITE_OK) - throwSQLiteError(db, "closing database"); - } catch (...) { - ignoreException(); - } -} - - -void SQLiteStmt::create(sqlite3 * db, const string & s) -{ - checkInterrupt(); - assert(!stmt); - if (sqlite3_prepare_v2(db, s.c_str(), -1, &stmt, 0) != SQLITE_OK) - throwSQLiteError(db, "creating statement"); - this->db = db; -} - - -void SQLiteStmt::reset() -{ - assert(stmt); - /* Note: sqlite3_reset() returns the error code for the most - recent call to sqlite3_step(). So ignore it. */ - sqlite3_reset(stmt); - curArg = 1; -} - - -SQLiteStmt::~SQLiteStmt() -{ - try { - if (stmt && sqlite3_finalize(stmt) != SQLITE_OK) - throwSQLiteError(db, "finalizing statement"); - } catch (...) { - ignoreException(); - } -} - - -void SQLiteStmt::bind(const string & value) -{ - if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) - throwSQLiteError(db, "binding argument"); -} - - -void SQLiteStmt::bind(int value) -{ - if (sqlite3_bind_int(stmt, curArg++, value) != SQLITE_OK) - throwSQLiteError(db, "binding argument"); -} - - -void SQLiteStmt::bind64(long long value) -{ - if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK) - throwSQLiteError(db, "binding argument"); -} - - -void SQLiteStmt::bind() -{ - if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK) - throwSQLiteError(db, "binding argument"); -} - - -/* Helper class to ensure that prepared statements are reset when - leaving the scope that uses them. Unfinished prepared statements - prevent transactions from being aborted, and can cause locks to be - kept when they should be released. */ -struct SQLiteStmtUse -{ - SQLiteStmt & stmt; - SQLiteStmtUse(SQLiteStmt & stmt) : stmt(stmt) - { - stmt.reset(); - } - ~SQLiteStmtUse() - { - try { - stmt.reset(); - } catch (...) { - ignoreException(); - } - } -}; - - -struct SQLiteTxn -{ - bool active; - sqlite3 * db; - - SQLiteTxn(sqlite3 * db) : active(false) { - this->db = db; - if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "starting transaction"); - active = true; - } - - void commit() - { - if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "committing transaction"); - active = false; - } - - ~SQLiteTxn() - { - try { - if (active && sqlite3_exec(db, "rollback;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "aborting transaction"); - } catch (...) { - ignoreException(); - } - } -}; - - -void checkStoreNotSymlink() -{ - if (getEnv("NIX_IGNORE_SYMLINK_STORE") == "1") return; - Path path = settings.nixStore; - struct stat st; - while (path != "/") { - if (lstat(path.c_str(), &st)) - throw SysError(format("getting status of `%1%'") % path); - if (S_ISLNK(st.st_mode)) - throw Error(format( - "the path `%1%' is a symlink; " - "this is not allowed for the Nix store and its parent directories") - % path); - path = dirOf(path); - } -} - - -LocalStore::LocalStore(bool reserveSpace) - : didSetSubstituterEnv(false) -{ - schemaPath = settings.nixDBPath + "/schema"; - - if (settings.readOnlyMode) { - openDB(false); - return; - } - - /* Create missing state directories if they don't already exist. */ - createDirs(settings.nixStore); - makeStoreWritable(); - createDirs(linksDir = settings.nixStore + "/.links"); - Path profilesDir = settings.nixStateDir + "/profiles"; - createDirs(settings.nixStateDir + "/profiles"); - createDirs(settings.nixStateDir + "/temproots"); - createDirs(settings.nixDBPath); - Path gcRootsDir = settings.nixStateDir + "/gcroots"; - if (!pathExists(gcRootsDir)) { - createDirs(gcRootsDir); - createSymlink(profilesDir, gcRootsDir + "/profiles"); - } - - checkStoreNotSymlink(); - - /* We can't open a SQLite database if the disk is full. Since - this prevents the garbage collector from running when it's most - needed, we reserve some dummy space that we can free just - before doing a garbage collection. */ - try { - Path reservedPath = settings.nixDBPath + "/reserved"; - if (reserveSpace) { - struct stat st; - if (stat(reservedPath.c_str(), &st) == -1 || - st.st_size != settings.reservedSize) - writeFile(reservedPath, string(settings.reservedSize, 'X')); - } - else - deletePath(reservedPath); - } catch (SysError & e) { /* don't care about errors */ - } - - /* Acquire the big fat lock in shared mode to make sure that no - schema upgrade is in progress. */ - try { - Path globalLockPath = settings.nixDBPath + "/big-lock"; - globalLock = openLockFile(globalLockPath.c_str(), true); - } catch (SysError & e) { - if (e.errNo != EACCES) throw; - settings.readOnlyMode = true; - openDB(false); - return; - } - - if (!lockFile(globalLock, ltRead, false)) { - printMsg(lvlError, "waiting for the big Nix store lock..."); - lockFile(globalLock, ltRead, true); - } - - /* Check the current database schema and if necessary do an - upgrade. */ - int curSchema = getSchema(); - if (curSchema > nixSchemaVersion) - throw Error(format("current Nix store schema is version %1%, but I only support %2%") - % curSchema % nixSchemaVersion); - - else if (curSchema == 0) { /* new store */ - curSchema = nixSchemaVersion; - openDB(true); - writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); - } - - else if (curSchema < nixSchemaVersion) { - if (curSchema < 5) - throw Error( - "Your Nix store has a database in Berkeley DB format,\n" - "which is no longer supported. To convert to the new format,\n" - "please upgrade Nix to version 0.12 first."); - - if (!lockFile(globalLock, ltWrite, false)) { - printMsg(lvlError, "waiting for exclusive access to the Nix store..."); - lockFile(globalLock, ltWrite, true); - } - - /* Get the schema version again, because another process may - have performed the upgrade already. */ - curSchema = getSchema(); - - if (curSchema < 6) upgradeStore6(); - else if (curSchema < 7) { upgradeStore7(); openDB(true); } - - writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); - - lockFile(globalLock, ltRead, true); - } - - else openDB(false); -} - - -LocalStore::~LocalStore() -{ - try { - foreach (RunningSubstituters::iterator, i, runningSubstituters) { - if (i->second.disabled) continue; - i->second.to.close(); - i->second.from.close(); - i->second.error.close(); - i->second.pid.wait(true); - } - } catch (...) { - ignoreException(); - } -} - - -int LocalStore::getSchema() -{ - int curSchema = 0; - if (pathExists(schemaPath)) { - string s = readFile(schemaPath); - if (!string2Int(s, curSchema)) - throw Error(format("`%1%' is corrupt") % schemaPath); - } - return curSchema; -} - - -void LocalStore::openDB(bool create) -{ - if (access(settings.nixDBPath.c_str(), R_OK | W_OK)) - throw SysError(format("Nix database directory `%1%' is not writable") % settings.nixDBPath); - - /* Open the Nix database. */ - string dbPath = settings.nixDBPath + "/db.sqlite"; - if (sqlite3_open_v2(dbPath.c_str(), &db.db, - SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK) - throw Error(format("cannot open Nix database `%1%'") % dbPath); - - if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) - throwSQLiteError(db, "setting timeout"); - - if (sqlite3_exec(db, "pragma foreign_keys = 1;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "enabling foreign keys"); - - /* !!! check whether sqlite has been built with foreign key - support */ - - /* Whether SQLite should fsync(). "Normal" synchronous mode - should be safe enough. If the user asks for it, don't sync at - all. This can cause database corruption if the system - crashes. */ - string syncMode = settings.fsyncMetadata ? "normal" : "off"; - if (sqlite3_exec(db, ("pragma synchronous = " + syncMode + ";").c_str(), 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "setting synchronous mode"); - - /* Set the SQLite journal mode. WAL mode is fastest, so it's the - default. */ - string mode = settings.useSQLiteWAL ? "wal" : "truncate"; - string prevMode; - { - SQLiteStmt stmt; - stmt.create(db, "pragma main.journal_mode;"); - if (sqlite3_step(stmt) != SQLITE_ROW) - throwSQLiteError(db, "querying journal mode"); - prevMode = string((const char *) sqlite3_column_text(stmt, 0)); - } - if (prevMode != mode && - sqlite3_exec(db, ("pragma main.journal_mode = " + mode + ";").c_str(), 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "setting journal mode"); - - /* Increase the auto-checkpoint interval to 40000 pages. This - seems enough to ensure that instantiating the NixOS system - derivation is done in a single fsync(). */ - if (mode == "wal" && sqlite3_exec(db, "pragma wal_autocheckpoint = 40000;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "setting autocheckpoint interval"); - - /* Initialise the database schema, if necessary. */ - if (create) { - const char * schema = -#include "schema.sql.hh" - ; - if (sqlite3_exec(db, (const char *) schema, 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "initialising database schema"); - } - - /* Prepare SQL statements. */ - stmtRegisterValidPath.create(db, - "insert into ValidPaths (path, hash, registrationTime, deriver, narSize) values (?, ?, ?, ?, ?);"); - stmtUpdatePathInfo.create(db, - "update ValidPaths set narSize = ?, hash = ? where path = ?;"); - stmtAddReference.create(db, - "insert or replace into Refs (referrer, reference) values (?, ?);"); - stmtQueryPathInfo.create(db, - "select id, hash, registrationTime, deriver, narSize from ValidPaths where path = ?;"); - stmtQueryReferences.create(db, - "select path from Refs join ValidPaths on reference = id where referrer = ?;"); - stmtQueryReferrers.create(db, - "select path from Refs join ValidPaths on referrer = id where reference = (select id from ValidPaths where path = ?);"); - stmtInvalidatePath.create(db, - "delete from ValidPaths where path = ?;"); - stmtRegisterFailedPath.create(db, - "insert or ignore into FailedPaths (path, time) values (?, ?);"); - stmtHasPathFailed.create(db, - "select time from FailedPaths where path = ?;"); - stmtQueryFailedPaths.create(db, - "select path from FailedPaths;"); - // If the path is a derivation, then clear its outputs. - stmtClearFailedPath.create(db, - "delete from FailedPaths where ?1 = '*' or path = ?1 " - "or path in (select d.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where v.path = ?1);"); - stmtAddDerivationOutput.create(db, - "insert or replace into DerivationOutputs (drv, id, path) values (?, ?, ?);"); - stmtQueryValidDerivers.create(db, - "select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where d.path = ?;"); - stmtQueryDerivationOutputs.create(db, - "select id, path from DerivationOutputs where drv = ?;"); - // Use "path >= ?" with limit 1 rather than "path like '?%'" to - // ensure efficient lookup. - stmtQueryPathFromHashPart.create(db, - "select path from ValidPaths where path >= ? limit 1;"); -} - - -/* To improve purity, users may want to make the Nix store a read-only - bind mount. So make the Nix store writable for this process. */ -void LocalStore::makeStoreWritable() -{ -#if HAVE_UNSHARE && HAVE_STATVFS && HAVE_SYS_MOUNT_H && defined(MS_BIND) && defined(MS_REMOUNT) - if (getuid() != 0) return; - /* Check if /nix/store is on a read-only mount. */ - struct statvfs stat; - if (statvfs(settings.nixStore.c_str(), &stat) != 0) - throw SysError("getting info about the Nix store mount point"); - - if (stat.f_flag & ST_RDONLY) { - if (unshare(CLONE_NEWNS) == -1) - throw SysError("setting up a private mount namespace"); - - if (mount(0, settings.nixStore.c_str(), 0, MS_REMOUNT | MS_BIND, 0) == -1) - throw SysError(format("remounting %1% writable") % settings.nixStore); - } -#endif -} - - -const time_t mtimeStore = 1; /* 1 second into the epoch */ - - -static void canonicaliseTimestampAndPermissions(const Path & path, const struct stat & st) -{ - if (!S_ISLNK(st.st_mode)) { - - /* Mask out all type related bits. */ - mode_t mode = st.st_mode & ~S_IFMT; - - if (mode != 0444 && mode != 0555) { - mode = (st.st_mode & S_IFMT) - | 0444 - | (st.st_mode & S_IXUSR ? 0111 : 0); - if (chmod(path.c_str(), mode) == -1) - throw SysError(format("changing mode of `%1%' to %2$o") % path % mode); - } - - } - - if (st.st_mtime != mtimeStore) { - struct timeval times[2]; - times[0].tv_sec = st.st_atime; - times[0].tv_usec = 0; - times[1].tv_sec = mtimeStore; - times[1].tv_usec = 0; -#if HAVE_LUTIMES - if (lutimes(path.c_str(), times) == -1) - if (errno != ENOSYS || - (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1)) -#else - if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1) -#endif - throw SysError(format("changing modification time of `%1%'") % path); - } -} - - -void canonicaliseTimestampAndPermissions(const Path & path) -{ - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path `%1%'") % path); - canonicaliseTimestampAndPermissions(path, st); -} - - -static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSeen & inodesSeen) -{ - checkInterrupt(); - - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path `%1%'") % path); - - /* Really make sure that the path is of a supported type. This - has already been checked in dumpPath(). */ - assert(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode)); - - /* Fail if the file is not owned by the build user. This prevents - us from messing up the ownership/permissions of files - hard-linked into the output (e.g. "ln /etc/shadow $out/foo"). - However, ignore files that we chown'ed ourselves previously to - ensure that we don't fail on hard links within the same build - (i.e. "touch $out/foo; ln $out/foo $out/bar"). */ - if (fromUid != (uid_t) -1 && st.st_uid != fromUid) { - assert(!S_ISDIR(st.st_mode)); - if (inodesSeen.find(Inode(st.st_dev, st.st_ino)) == inodesSeen.end()) - throw BuildError(format("invalid ownership on file `%1%'") % path); - mode_t mode = st.st_mode & ~S_IFMT; - assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore)); - return; - } - - inodesSeen.insert(Inode(st.st_dev, st.st_ino)); - - canonicaliseTimestampAndPermissions(path, st); - - /* Change ownership to the current uid. If it's a symlink, use - lchown if available, otherwise don't bother. Wrong ownership - of a symlink doesn't matter, since the owning user can't change - the symlink and can't delete it because the directory is not - writable. The only exception is top-level paths in the Nix - store (since that directory is group-writable for the Nix build - users group); we check for this case below. */ - if (st.st_uid != geteuid()) { -#if HAVE_LCHOWN - if (lchown(path.c_str(), geteuid(), (gid_t) -1) == -1) -#else - if (!S_ISLNK(st.st_mode) && - chown(path.c_str(), geteuid(), (gid_t) -1) == -1) -#endif - throw SysError(format("changing owner of `%1%' to %2%") - % path % geteuid()); - } - - if (S_ISDIR(st.st_mode)) { - Strings names = readDirectory(path); - foreach (Strings::iterator, i, names) - canonicalisePathMetaData_(path + "/" + *i, fromUid, inodesSeen); - } -} - - -void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & inodesSeen) -{ - canonicalisePathMetaData_(path, fromUid, inodesSeen); - - /* On platforms that don't have lchown(), the top-level path can't - be a symlink, since we can't change its ownership. */ - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path `%1%'") % path); - - if (st.st_uid != geteuid()) { - assert(S_ISLNK(st.st_mode)); - throw Error(format("wrong ownership of top-level store path `%1%'") % path); - } -} - - -void canonicalisePathMetaData(const Path & path, uid_t fromUid) -{ - InodesSeen inodesSeen; - canonicalisePathMetaData(path, fromUid, inodesSeen); -} - - -void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation & drv) -{ - string drvName = storePathToName(drvPath); - assert(isDerivation(drvName)); - drvName = string(drvName, 0, drvName.size() - drvExtension.size()); - - if (isFixedOutputDrv(drv)) { - DerivationOutputs::const_iterator out = drv.outputs.find("out"); - if (out == drv.outputs.end()) - throw Error(format("derivation `%1%' does not have an output named `out'") % drvPath); - - bool recursive; HashType ht; Hash h; - out->second.parseHashInfo(recursive, ht, h); - Path outPath = makeFixedOutputPath(recursive, ht, h, drvName); - - StringPairs::const_iterator j = drv.env.find("out"); - if (out->second.path != outPath || j == drv.env.end() || j->second != outPath) - throw Error(format("derivation `%1%' has incorrect output `%2%', should be `%3%'") - % drvPath % out->second.path % outPath); - } - - else { - Derivation drvCopy(drv); - foreach (DerivationOutputs::iterator, i, drvCopy.outputs) { - i->second.path = ""; - drvCopy.env[i->first] = ""; - } - - Hash h = hashDerivationModulo(*this, drvCopy); - - foreach (DerivationOutputs::const_iterator, i, drv.outputs) { - Path outPath = makeOutputPath(i->first, h, drvName); - StringPairs::const_iterator j = drv.env.find(i->first); - if (i->second.path != outPath || j == drv.env.end() || j->second != outPath) - throw Error(format("derivation `%1%' has incorrect output `%2%', should be `%3%'") - % drvPath % i->second.path % outPath); - } - } -} - - -unsigned long long LocalStore::addValidPath(const ValidPathInfo & info, bool checkOutputs) -{ - SQLiteStmtUse use(stmtRegisterValidPath); - stmtRegisterValidPath.bind(info.path); - stmtRegisterValidPath.bind("sha256:" + printHash(info.hash)); - stmtRegisterValidPath.bind(info.registrationTime == 0 ? time(0) : info.registrationTime); - if (info.deriver != "") - stmtRegisterValidPath.bind(info.deriver); - else - stmtRegisterValidPath.bind(); // null - if (info.narSize != 0) - stmtRegisterValidPath.bind64(info.narSize); - else - stmtRegisterValidPath.bind(); // null - if (sqlite3_step(stmtRegisterValidPath) != SQLITE_DONE) - throwSQLiteError(db, format("registering valid path `%1%' in database") % info.path); - unsigned long long id = sqlite3_last_insert_rowid(db); - - /* If this is a derivation, then store the derivation outputs in - the database. This is useful for the garbage collector: it can - efficiently query whether a path is an output of some - derivation. */ - if (isDerivation(info.path)) { - Derivation drv = parseDerivation(readFile(info.path)); - - /* Verify that the output paths in the derivation are correct - (i.e., follow the scheme for computing output paths from - derivations). Note that if this throws an error, then the - DB transaction is rolled back, so the path validity - registration above is undone. */ - if (checkOutputs) checkDerivationOutputs(info.path, drv); - - foreach (DerivationOutputs::iterator, i, drv.outputs) { - SQLiteStmtUse use(stmtAddDerivationOutput); - stmtAddDerivationOutput.bind(id); - stmtAddDerivationOutput.bind(i->first); - stmtAddDerivationOutput.bind(i->second.path); - if (sqlite3_step(stmtAddDerivationOutput) != SQLITE_DONE) - throwSQLiteError(db, format("adding derivation output for `%1%' in database") % info.path); - } - } - - return id; -} - - -void LocalStore::addReference(unsigned long long referrer, unsigned long long reference) -{ - SQLiteStmtUse use(stmtAddReference); - stmtAddReference.bind(referrer); - stmtAddReference.bind(reference); - if (sqlite3_step(stmtAddReference) != SQLITE_DONE) - throwSQLiteError(db, "adding reference to database"); -} - - -void LocalStore::registerFailedPath(const Path & path) -{ - retry_sqlite { - SQLiteStmtUse use(stmtRegisterFailedPath); - stmtRegisterFailedPath.bind(path); - stmtRegisterFailedPath.bind(time(0)); - if (sqlite3_step(stmtRegisterFailedPath) != SQLITE_DONE) - throwSQLiteError(db, format("registering failed path `%1%'") % path); - } end_retry_sqlite; -} - - -bool LocalStore::hasPathFailed(const Path & path) -{ - retry_sqlite { - SQLiteStmtUse use(stmtHasPathFailed); - stmtHasPathFailed.bind(path); - int res = sqlite3_step(stmtHasPathFailed); - if (res != SQLITE_DONE && res != SQLITE_ROW) - throwSQLiteError(db, "querying whether path failed"); - return res == SQLITE_ROW; - } end_retry_sqlite; -} - - -PathSet LocalStore::queryFailedPaths() -{ - retry_sqlite { - SQLiteStmtUse use(stmtQueryFailedPaths); - - PathSet res; - int r; - while ((r = sqlite3_step(stmtQueryFailedPaths)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryFailedPaths, 0); - assert(s); - res.insert(s); - } - - if (r != SQLITE_DONE) - throwSQLiteError(db, "error querying failed paths"); - - return res; - } end_retry_sqlite; -} - - -void LocalStore::clearFailedPaths(const PathSet & paths) -{ - retry_sqlite { - SQLiteTxn txn(db); - - foreach (PathSet::const_iterator, i, paths) { - SQLiteStmtUse use(stmtClearFailedPath); - stmtClearFailedPath.bind(*i); - if (sqlite3_step(stmtClearFailedPath) != SQLITE_DONE) - throwSQLiteError(db, format("clearing failed path `%1%' in database") % *i); - } - - txn.commit(); - } end_retry_sqlite; -} - - -Hash parseHashField(const Path & path, const string & s) -{ - string::size_type colon = s.find(':'); - if (colon == string::npos) - throw Error(format("corrupt hash `%1%' in valid-path entry for `%2%'") - % s % path); - HashType ht = parseHashType(string(s, 0, colon)); - if (ht == htUnknown) - throw Error(format("unknown hash type `%1%' in valid-path entry for `%2%'") - % string(s, 0, colon) % path); - return parseHash(ht, string(s, colon + 1)); -} - - -ValidPathInfo LocalStore::queryPathInfo(const Path & path) -{ - ValidPathInfo info; - info.path = path; - - assertStorePath(path); - - retry_sqlite { - - /* Get the path info. */ - SQLiteStmtUse use1(stmtQueryPathInfo); - - stmtQueryPathInfo.bind(path); - - int r = sqlite3_step(stmtQueryPathInfo); - if (r == SQLITE_DONE) throw Error(format("path `%1%' is not valid") % path); - if (r != SQLITE_ROW) throwSQLiteError(db, "querying path in database"); - - info.id = sqlite3_column_int(stmtQueryPathInfo, 0); - - const char * s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 1); - assert(s); - info.hash = parseHashField(path, s); - - info.registrationTime = sqlite3_column_int(stmtQueryPathInfo, 2); - - s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 3); - if (s) info.deriver = s; - - /* Note that narSize = NULL yields 0. */ - info.narSize = sqlite3_column_int64(stmtQueryPathInfo, 4); - - /* Get the references. */ - SQLiteStmtUse use2(stmtQueryReferences); - - stmtQueryReferences.bind(info.id); - - while ((r = sqlite3_step(stmtQueryReferences)) == SQLITE_ROW) { - s = (const char *) sqlite3_column_text(stmtQueryReferences, 0); - assert(s); - info.references.insert(s); - } - - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting references of `%1%'") % path); - - return info; - } end_retry_sqlite; -} - - -/* Update path info in the database. Currently only updates the - narSize field. */ -void LocalStore::updatePathInfo(const ValidPathInfo & info) -{ - SQLiteStmtUse use(stmtUpdatePathInfo); - if (info.narSize != 0) - stmtUpdatePathInfo.bind64(info.narSize); - else - stmtUpdatePathInfo.bind(); // null - stmtUpdatePathInfo.bind("sha256:" + printHash(info.hash)); - stmtUpdatePathInfo.bind(info.path); - if (sqlite3_step(stmtUpdatePathInfo) != SQLITE_DONE) - throwSQLiteError(db, format("updating info of path `%1%' in database") % info.path); -} - - -unsigned long long LocalStore::queryValidPathId(const Path & path) -{ - SQLiteStmtUse use(stmtQueryPathInfo); - stmtQueryPathInfo.bind(path); - int res = sqlite3_step(stmtQueryPathInfo); - if (res == SQLITE_ROW) return sqlite3_column_int(stmtQueryPathInfo, 0); - if (res == SQLITE_DONE) throw Error(format("path `%1%' is not valid") % path); - throwSQLiteError(db, "querying path in database"); -} - - -bool LocalStore::isValidPath_(const Path & path) -{ - SQLiteStmtUse use(stmtQueryPathInfo); - stmtQueryPathInfo.bind(path); - int res = sqlite3_step(stmtQueryPathInfo); - if (res != SQLITE_DONE && res != SQLITE_ROW) - throwSQLiteError(db, "querying path in database"); - return res == SQLITE_ROW; -} - - -bool LocalStore::isValidPath(const Path & path) -{ - retry_sqlite { - return isValidPath_(path); - } end_retry_sqlite; -} - - -PathSet LocalStore::queryValidPaths(const PathSet & paths) -{ - retry_sqlite { - PathSet res; - foreach (PathSet::const_iterator, i, paths) - if (isValidPath_(*i)) res.insert(*i); - return res; - } end_retry_sqlite; -} - - -PathSet LocalStore::queryAllValidPaths() -{ - retry_sqlite { - SQLiteStmt stmt; - stmt.create(db, "select path from ValidPaths"); - - PathSet res; - int r; - while ((r = sqlite3_step(stmt)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmt, 0); - assert(s); - res.insert(s); - } - - if (r != SQLITE_DONE) - throwSQLiteError(db, "error getting valid paths"); - - return res; - } end_retry_sqlite; -} - - -void LocalStore::queryReferences(const Path & path, - PathSet & references) -{ - ValidPathInfo info = queryPathInfo(path); - references.insert(info.references.begin(), info.references.end()); -} - - -void LocalStore::queryReferrers_(const Path & path, PathSet & referrers) -{ - SQLiteStmtUse use(stmtQueryReferrers); - - stmtQueryReferrers.bind(path); - - int r; - while ((r = sqlite3_step(stmtQueryReferrers)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryReferrers, 0); - assert(s); - referrers.insert(s); - } - - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting references of `%1%'") % path); -} - - -void LocalStore::queryReferrers(const Path & path, PathSet & referrers) -{ - assertStorePath(path); - retry_sqlite { - queryReferrers_(path, referrers); - } end_retry_sqlite; -} - - -Path LocalStore::queryDeriver(const Path & path) -{ - return queryPathInfo(path).deriver; -} - - -PathSet LocalStore::queryValidDerivers(const Path & path) -{ - assertStorePath(path); - - retry_sqlite { - SQLiteStmtUse use(stmtQueryValidDerivers); - stmtQueryValidDerivers.bind(path); - - PathSet derivers; - int r; - while ((r = sqlite3_step(stmtQueryValidDerivers)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryValidDerivers, 1); - assert(s); - derivers.insert(s); - } - - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting valid derivers of `%1%'") % path); - - return derivers; - } end_retry_sqlite; -} - - -PathSet LocalStore::queryDerivationOutputs(const Path & path) -{ - retry_sqlite { - SQLiteStmtUse use(stmtQueryDerivationOutputs); - stmtQueryDerivationOutputs.bind(queryValidPathId(path)); - - PathSet outputs; - int r; - while ((r = sqlite3_step(stmtQueryDerivationOutputs)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryDerivationOutputs, 1); - assert(s); - outputs.insert(s); - } - - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting outputs of `%1%'") % path); - - return outputs; - } end_retry_sqlite; -} - - -StringSet LocalStore::queryDerivationOutputNames(const Path & path) -{ - retry_sqlite { - SQLiteStmtUse use(stmtQueryDerivationOutputs); - stmtQueryDerivationOutputs.bind(queryValidPathId(path)); - - StringSet outputNames; - int r; - while ((r = sqlite3_step(stmtQueryDerivationOutputs)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryDerivationOutputs, 0); - assert(s); - outputNames.insert(s); - } - - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting output names of `%1%'") % path); - - return outputNames; - } end_retry_sqlite; -} - - -Path LocalStore::queryPathFromHashPart(const string & hashPart) -{ - if (hashPart.size() != 32) throw Error("invalid hash part"); - - Path prefix = settings.nixStore + "/" + hashPart; - - retry_sqlite { - SQLiteStmtUse use(stmtQueryPathFromHashPart); - stmtQueryPathFromHashPart.bind(prefix); - - int res = sqlite3_step(stmtQueryPathFromHashPart); - if (res == SQLITE_DONE) return ""; - if (res != SQLITE_ROW) throwSQLiteError(db, "finding path in database"); - - const char * s = (const char *) sqlite3_column_text(stmtQueryPathFromHashPart, 0); - return s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0 ? s : ""; - } end_retry_sqlite; -} - - -void LocalStore::setSubstituterEnv() -{ - if (didSetSubstituterEnv) return; - - /* Pass configuration options (including those overridden with - --option) to substituters. */ - setenv("_NIX_OPTIONS", settings.pack().c_str(), 1); - - didSetSubstituterEnv = true; -} - - -void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter & run) -{ - if (run.disabled || run.pid != -1) return; - - debug(format("starting substituter program `%1%'") % substituter); - - Pipe toPipe, fromPipe, errorPipe; - - toPipe.create(); - fromPipe.create(); - errorPipe.create(); - - setSubstituterEnv(); - - run.pid = maybeVfork(); - - switch (run.pid) { - - case -1: - throw SysError("unable to fork"); - - case 0: /* child */ - try { - restoreAffinity(); - if (dup2(toPipe.readSide, STDIN_FILENO) == -1) - throw SysError("dupping stdin"); - if (dup2(fromPipe.writeSide, STDOUT_FILENO) == -1) - throw SysError("dupping stdout"); - if (dup2(errorPipe.writeSide, STDERR_FILENO) == -1) - throw SysError("dupping stderr"); - execl(substituter.c_str(), substituter.c_str(), "--query", NULL); - throw SysError(format("executing `%1%'") % substituter); - } catch (std::exception & e) { - std::cerr << "error: " << e.what() << std::endl; - } - _exit(1); - } - - /* Parent. */ - - run.program = baseNameOf(substituter); - run.to = toPipe.writeSide.borrow(); - run.from = run.fromBuf.fd = fromPipe.readSide.borrow(); - run.error = errorPipe.readSide.borrow(); - - toPipe.readSide.close(); - fromPipe.writeSide.close(); - errorPipe.writeSide.close(); - - /* The substituter may exit right away if it's disabled in any way - (e.g. copy-from-other-stores.pl will exit if no other stores - are configured). */ - try { - getLineFromSubstituter(run); - } catch (EndOfFile & e) { - run.to.close(); - run.from.close(); - run.error.close(); - run.disabled = true; - if (run.pid.wait(true) != 0) throw; - } -} - - -/* Read a line from the substituter's stdout, while also processing - its stderr. */ -string LocalStore::getLineFromSubstituter(RunningSubstituter & run) -{ - string res, err; - - /* We might have stdout data left over from the last time. */ - if (run.fromBuf.hasData()) goto haveData; - - while (1) { - checkInterrupt(); - - fd_set fds; - FD_ZERO(&fds); - FD_SET(run.from, &fds); - FD_SET(run.error, &fds); - - /* Wait for data to appear on the substituter's stdout or - stderr. */ - if (select(run.from > run.error ? run.from + 1 : run.error + 1, &fds, 0, 0, 0) == -1) { - if (errno == EINTR) continue; - throw SysError("waiting for input from the substituter"); - } - - /* Completely drain stderr before dealing with stdout. */ - if (FD_ISSET(run.error, &fds)) { - char buf[4096]; - ssize_t n = read(run.error, (unsigned char *) buf, sizeof(buf)); - if (n == -1) { - if (errno == EINTR) continue; - throw SysError("reading from substituter's stderr"); - } - if (n == 0) throw EndOfFile(format("substituter `%1%' died unexpectedly") % run.program); - err.append(buf, n); - string::size_type p; - while ((p = err.find('\n')) != string::npos) { - printMsg(lvlError, run.program + ": " + string(err, 0, p)); - err = string(err, p + 1); - } - } - - /* Read from stdout until we get a newline or the buffer is empty. */ - else if (run.fromBuf.hasData() || FD_ISSET(run.from, &fds)) { - haveData: - do { - unsigned char c; - run.fromBuf(&c, 1); - if (c == '\n') { - if (!err.empty()) printMsg(lvlError, run.program + ": " + err); - return res; - } - res += c; - } while (run.fromBuf.hasData()); - } - } -} - - -template<class T> T LocalStore::getIntLineFromSubstituter(RunningSubstituter & run) -{ - string s = getLineFromSubstituter(run); - T res; - if (!string2Int(s, res)) throw Error("integer expected from stream"); - return res; -} - - -PathSet LocalStore::querySubstitutablePaths(const PathSet & paths) -{ - PathSet res; - foreach (Paths::iterator, i, settings.substituters) { - if (res.size() == paths.size()) break; - RunningSubstituter & run(runningSubstituters[*i]); - startSubstituter(*i, run); - if (run.disabled) continue; - string s = "have "; - foreach (PathSet::const_iterator, j, paths) - if (res.find(*j) == res.end()) { s += *j; s += " "; } - writeLine(run.to, s); - while (true) { - /* FIXME: we only read stderr when an error occurs, so - substituters should only write (short) messages to - stderr when they fail. I.e. they shouldn't write debug - output. */ - Path path = getLineFromSubstituter(run); - if (path == "") break; - res.insert(path); - } - } - return res; -} - - -void LocalStore::querySubstitutablePathInfos(const Path & substituter, - PathSet & paths, SubstitutablePathInfos & infos) -{ - RunningSubstituter & run(runningSubstituters[substituter]); - startSubstituter(substituter, run); - if (run.disabled) return; - - string s = "info "; - foreach (PathSet::const_iterator, i, paths) - if (infos.find(*i) == infos.end()) { s += *i; s += " "; } - writeLine(run.to, s); - - while (true) { - Path path = getLineFromSubstituter(run); - if (path == "") break; - if (paths.find(path) == paths.end()) - throw Error(format("got unexpected path `%1%' from substituter") % path); - paths.erase(path); - SubstitutablePathInfo & info(infos[path]); - info.deriver = getLineFromSubstituter(run); - if (info.deriver != "") assertStorePath(info.deriver); - int nrRefs = getIntLineFromSubstituter<int>(run); - while (nrRefs--) { - Path p = getLineFromSubstituter(run); - assertStorePath(p); - info.references.insert(p); - } - info.downloadSize = getIntLineFromSubstituter<long long>(run); - info.narSize = getIntLineFromSubstituter<long long>(run); - } -} - - -void LocalStore::querySubstitutablePathInfos(const PathSet & paths, - SubstitutablePathInfos & infos) -{ - PathSet todo = paths; - foreach (Paths::iterator, i, settings.substituters) { - if (todo.empty()) break; - querySubstitutablePathInfos(*i, todo, infos); - } -} - - -Hash LocalStore::queryPathHash(const Path & path) -{ - return queryPathInfo(path).hash; -} - - -void LocalStore::registerValidPath(const ValidPathInfo & info) -{ - ValidPathInfos infos; - infos.push_back(info); - registerValidPaths(infos); -} - - -void LocalStore::registerValidPaths(const ValidPathInfos & infos) -{ - /* SQLite will fsync by default, but the new valid paths may not be fsync-ed. - * So some may want to fsync them before registering the validity, at the - * expense of some speed of the path registering operation. */ - if (settings.syncBeforeRegistering) sync(); - - retry_sqlite { - SQLiteTxn txn(db); - PathSet paths; - - foreach (ValidPathInfos::const_iterator, i, infos) { - assert(i->hash.type == htSHA256); - if (isValidPath_(i->path)) - updatePathInfo(*i); - else - addValidPath(*i, false); - paths.insert(i->path); - } - - foreach (ValidPathInfos::const_iterator, i, infos) { - unsigned long long referrer = queryValidPathId(i->path); - foreach (PathSet::iterator, j, i->references) - addReference(referrer, queryValidPathId(*j)); - } - - /* Check that the derivation outputs are correct. We can't do - this in addValidPath() above, because the references might - not be valid yet. */ - foreach (ValidPathInfos::const_iterator, i, infos) - if (isDerivation(i->path)) { - // FIXME: inefficient; we already loaded the - // derivation in addValidPath(). - Derivation drv = parseDerivation(readFile(i->path)); - checkDerivationOutputs(i->path, drv); - } - - /* Do a topological sort of the paths. This will throw an - error if a cycle is detected and roll back the - transaction. Cycles can only occur when a derivation - has multiple outputs. */ - topoSortPaths(*this, paths); - - txn.commit(); - } end_retry_sqlite; -} - - -/* Invalidate a path. The caller is responsible for checking that - there are no referrers. */ -void LocalStore::invalidatePath(const Path & path) -{ - debug(format("invalidating path `%1%'") % path); - - drvHashes.erase(path); - - SQLiteStmtUse use(stmtInvalidatePath); - - stmtInvalidatePath.bind(path); - - if (sqlite3_step(stmtInvalidatePath) != SQLITE_DONE) - throwSQLiteError(db, format("invalidating path `%1%' in database") % path); - - /* Note that the foreign key constraints on the Refs table take - care of deleting the references entries for `path'. */ -} - - -Path LocalStore::addToStoreFromDump(const string & dump, const string & name, - bool recursive, HashType hashAlgo, bool repair) -{ - Hash h = hashString(hashAlgo, dump); - - Path dstPath = makeFixedOutputPath(recursive, hashAlgo, h, name); - - addTempRoot(dstPath); - - if (repair || !isValidPath(dstPath)) { - - /* The first check above is an optimisation to prevent - unnecessary lock acquisition. */ - - PathLocks outputLock(singleton<PathSet, Path>(dstPath)); - - if (repair || !isValidPath(dstPath)) { - - if (pathExists(dstPath)) deletePath(dstPath); - - if (recursive) { - StringSource source(dump); - restorePath(dstPath, source); - } else - writeFile(dstPath, dump); - - canonicalisePathMetaData(dstPath, -1); - - /* Register the SHA-256 hash of the NAR serialisation of - the path in the database. We may just have computed it - above (if called with recursive == true and hashAlgo == - sha256); otherwise, compute it here. */ - HashResult hash; - if (recursive) { - hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump); - hash.second = dump.size(); - } else - hash = hashPath(htSHA256, dstPath); - - optimisePath(dstPath); // FIXME: combine with hashPath() - - ValidPathInfo info; - info.path = dstPath; - info.hash = hash.first; - info.narSize = hash.second; - registerValidPath(info); - } - - outputLock.setDeletion(true); - } - - return dstPath; -} - - -Path LocalStore::addToStore(const Path & _srcPath, - bool recursive, HashType hashAlgo, PathFilter & filter, bool repair) -{ - Path srcPath(absPath(_srcPath)); - debug(format("adding `%1%' to the store") % srcPath); - - /* Read the whole path into memory. This is not a very scalable - method for very large paths, but `copyPath' is mainly used for - small files. */ - StringSink sink; - if (recursive) - dumpPath(srcPath, sink, filter); - else - sink.s = readFile(srcPath); - - return addToStoreFromDump(sink.s, baseNameOf(srcPath), recursive, hashAlgo, repair); -} - - -Path LocalStore::addTextToStore(const string & name, const string & s, - const PathSet & references, bool repair) -{ - Path dstPath = computeStorePathForText(name, s, references); - - addTempRoot(dstPath); - - if (repair || !isValidPath(dstPath)) { - - PathLocks outputLock(singleton<PathSet, Path>(dstPath)); - - if (repair || !isValidPath(dstPath)) { - - if (pathExists(dstPath)) deletePath(dstPath); - - writeFile(dstPath, s); - - canonicalisePathMetaData(dstPath, -1); - - HashResult hash = hashPath(htSHA256, dstPath); - - optimisePath(dstPath); - - ValidPathInfo info; - info.path = dstPath; - info.hash = hash.first; - info.narSize = hash.second; - info.references = references; - registerValidPath(info); - } - - outputLock.setDeletion(true); - } - - return dstPath; -} - - -struct HashAndWriteSink : Sink -{ - Sink & writeSink; - HashSink hashSink; - HashAndWriteSink(Sink & writeSink) : writeSink(writeSink), hashSink(htSHA256) - { - } - virtual void operator () (const unsigned char * data, size_t len) - { - writeSink(data, len); - hashSink(data, len); - } - Hash currentHash() - { - return hashSink.currentHash().first; - } -}; - - -#define EXPORT_MAGIC 0x4558494e - - -static void checkSecrecy(const Path & path) -{ - struct stat st; - if (stat(path.c_str(), &st)) - throw SysError(format("getting status of `%1%'") % path); - if ((st.st_mode & (S_IRWXG | S_IRWXO)) != 0) - throw Error(format("file `%1%' should be secret (inaccessible to everybody else)!") % path); -} - - -void LocalStore::exportPath(const Path & path, bool sign, - Sink & sink) -{ - assertStorePath(path); - - addTempRoot(path); - if (!isValidPath(path)) - throw Error(format("path `%1%' is not valid") % path); - - HashAndWriteSink hashAndWriteSink(sink); - - dumpPath(path, hashAndWriteSink); - - /* Refuse to export paths that have changed. This prevents - filesystem corruption from spreading to other machines. - Don't complain if the stored hash is zero (unknown). */ - Hash hash = hashAndWriteSink.currentHash(); - Hash storedHash = queryPathHash(path); - if (hash != storedHash && storedHash != Hash(storedHash.type)) - throw Error(format("hash of path `%1%' has changed from `%2%' to `%3%'!") % path - % printHash(storedHash) % printHash(hash)); - - writeInt(EXPORT_MAGIC, hashAndWriteSink); - - writeString(path, hashAndWriteSink); - - PathSet references; - queryReferences(path, references); - writeStrings(references, hashAndWriteSink); - - Path deriver = queryDeriver(path); - writeString(deriver, hashAndWriteSink); - - if (sign) { - Hash hash = hashAndWriteSink.currentHash(); - - writeInt(1, hashAndWriteSink); - - Path tmpDir = createTempDir(); - AutoDelete delTmp(tmpDir); - Path hashFile = tmpDir + "/hash"; - writeFile(hashFile, printHash(hash)); - - Path secretKey = settings.nixConfDir + "/signing-key.sec"; - checkSecrecy(secretKey); - - Strings args; - args.push_back("rsautl"); - args.push_back("-sign"); - args.push_back("-inkey"); - args.push_back(secretKey); - args.push_back("-in"); - args.push_back(hashFile); - string signature = runProgram(OPENSSL_PATH, true, args); - - writeString(signature, hashAndWriteSink); - - } else - writeInt(0, hashAndWriteSink); -} - - -struct HashAndReadSource : Source -{ - Source & readSource; - HashSink hashSink; - bool hashing; - HashAndReadSource(Source & readSource) : readSource(readSource), hashSink(htSHA256) - { - hashing = true; - } - size_t read(unsigned char * data, size_t len) - { - size_t n = readSource.read(data, len); - if (hashing) hashSink(data, n); - return n; - } -}; - - -/* Create a temporary directory in the store that won't be - garbage-collected. */ -Path LocalStore::createTempDirInStore() -{ - Path tmpDir; - do { - /* There is a slight possibility that `tmpDir' gets deleted by - the GC between createTempDir() and addTempRoot(), so repeat - until `tmpDir' exists. */ - tmpDir = createTempDir(settings.nixStore); - addTempRoot(tmpDir); - } while (!pathExists(tmpDir)); - return tmpDir; -} - - -Path LocalStore::importPath(bool requireSignature, Source & source) -{ - HashAndReadSource hashAndReadSource(source); - - /* We don't yet know what store path this archive contains (the - store path follows the archive data proper), and besides, we - don't know yet whether the signature is valid. */ - Path tmpDir = createTempDirInStore(); - AutoDelete delTmp(tmpDir); - Path unpacked = tmpDir + "/unpacked"; - - restorePath(unpacked, hashAndReadSource); - - unsigned int magic = readInt(hashAndReadSource); - if (magic != EXPORT_MAGIC) - throw Error("Nix archive cannot be imported; wrong format"); - - Path dstPath = readStorePath(hashAndReadSource); - - printMsg(lvlInfo, format("importing path `%1%'") % dstPath); - - PathSet references = readStorePaths<PathSet>(hashAndReadSource); - - Path deriver = readString(hashAndReadSource); - if (deriver != "") assertStorePath(deriver); - - Hash hash = hashAndReadSource.hashSink.finish().first; - hashAndReadSource.hashing = false; - - bool haveSignature = readInt(hashAndReadSource) == 1; - - if (requireSignature && !haveSignature) - throw Error(format("imported archive of `%1%' lacks a signature") % dstPath); - - if (haveSignature) { - string signature = readString(hashAndReadSource); - - if (requireSignature) { - Path sigFile = tmpDir + "/sig"; - writeFile(sigFile, signature); - - Strings args; - args.push_back("rsautl"); - args.push_back("-verify"); - args.push_back("-inkey"); - args.push_back(settings.nixConfDir + "/signing-key.pub"); - args.push_back("-pubin"); - args.push_back("-in"); - args.push_back(sigFile); - string hash2 = runProgram(OPENSSL_PATH, true, args); - - /* Note: runProgram() throws an exception if the signature - is invalid. */ - - if (printHash(hash) != hash2) - throw Error( - "signed hash doesn't match actual contents of imported " - "archive; archive could be corrupt, or someone is trying " - "to import a Trojan horse"); - } - } - - /* Do the actual import. */ - - /* !!! way too much code duplication with addTextToStore() etc. */ - addTempRoot(dstPath); - - if (!isValidPath(dstPath)) { - - PathLocks outputLock; - - /* Lock the output path. But don't lock if we're being called - from a build hook (whose parent process already acquired a - lock on this path). */ - Strings locksHeld = tokenizeString<Strings>(getEnv("NIX_HELD_LOCKS")); - if (find(locksHeld.begin(), locksHeld.end(), dstPath) == locksHeld.end()) - outputLock.lockPaths(singleton<PathSet, Path>(dstPath)); - - if (!isValidPath(dstPath)) { - - if (pathExists(dstPath)) deletePath(dstPath); - - if (rename(unpacked.c_str(), dstPath.c_str()) == -1) - throw SysError(format("cannot move `%1%' to `%2%'") - % unpacked % dstPath); - - canonicalisePathMetaData(dstPath, -1); - - /* !!! if we were clever, we could prevent the hashPath() - here. */ - HashResult hash = hashPath(htSHA256, dstPath); - - optimisePath(dstPath); // FIXME: combine with hashPath() - - ValidPathInfo info; - info.path = dstPath; - info.hash = hash.first; - info.narSize = hash.second; - info.references = references; - info.deriver = deriver != "" && isValidPath(deriver) ? deriver : ""; - registerValidPath(info); - } - - outputLock.setDeletion(true); - } - - return dstPath; -} - - -Paths LocalStore::importPaths(bool requireSignature, Source & source) -{ - Paths res; - while (true) { - unsigned long long n = readLongLong(source); - if (n == 0) break; - if (n != 1) throw Error("input doesn't look like something created by `nix-store --export'"); - res.push_back(importPath(requireSignature, source)); - } - return res; -} - - -void LocalStore::invalidatePathChecked(const Path & path) -{ - assertStorePath(path); - - retry_sqlite { - SQLiteTxn txn(db); - - if (isValidPath_(path)) { - PathSet referrers; queryReferrers_(path, referrers); - referrers.erase(path); /* ignore self-references */ - if (!referrers.empty()) - throw PathInUse(format("cannot delete path `%1%' because it is in use by %2%") - % path % showPaths(referrers)); - invalidatePath(path); - } - - txn.commit(); - } end_retry_sqlite; -} - - -bool LocalStore::verifyStore(bool checkContents, bool repair) -{ - printMsg(lvlError, format("reading the Nix store...")); - - bool errors = false; - - /* Acquire the global GC lock to prevent a garbage collection. */ - AutoCloseFD fdGCLock = openGCLock(ltWrite); - - Paths entries = readDirectory(settings.nixStore); - PathSet store(entries.begin(), entries.end()); - - /* Check whether all valid paths actually exist. */ - printMsg(lvlInfo, "checking path existence..."); - - PathSet validPaths2 = queryAllValidPaths(), validPaths, done; - - foreach (PathSet::iterator, i, validPaths2) - verifyPath(*i, store, done, validPaths, repair, errors); - - /* Release the GC lock so that checking content hashes (which can - take ages) doesn't block the GC or builds. */ - fdGCLock.close(); - - /* Optionally, check the content hashes (slow). */ - if (checkContents) { - printMsg(lvlInfo, "checking hashes..."); - - Hash nullHash(htSHA256); - - foreach (PathSet::iterator, i, validPaths) { - try { - ValidPathInfo info = queryPathInfo(*i); - - /* Check the content hash (optionally - slow). */ - printMsg(lvlTalkative, format("checking contents of `%1%'") % *i); - HashResult current = hashPath(info.hash.type, *i); - - if (info.hash != nullHash && info.hash != current.first) { - printMsg(lvlError, format("path `%1%' was modified! " - "expected hash `%2%', got `%3%'") - % *i % printHash(info.hash) % printHash(current.first)); - if (repair) repairPath(*i); else errors = true; - } else { - - bool update = false; - - /* Fill in missing hashes. */ - if (info.hash == nullHash) { - printMsg(lvlError, format("fixing missing hash on `%1%'") % *i); - info.hash = current.first; - update = true; - } - - /* Fill in missing narSize fields (from old stores). */ - if (info.narSize == 0) { - printMsg(lvlError, format("updating size field on `%1%' to %2%") % *i % current.second); - info.narSize = current.second; - update = true; - } - - if (update) updatePathInfo(info); - - } - - } catch (Error & e) { - /* It's possible that the path got GC'ed, so ignore - errors on invalid paths. */ - if (isValidPath(*i)) - printMsg(lvlError, format("error: %1%") % e.msg()); - else - printMsg(lvlError, format("warning: %1%") % e.msg()); - errors = true; - } - } - } - - return errors; -} - - -void LocalStore::verifyPath(const Path & path, const PathSet & store, - PathSet & done, PathSet & validPaths, bool repair, bool & errors) -{ - checkInterrupt(); - - if (done.find(path) != done.end()) return; - done.insert(path); - - if (!isStorePath(path)) { - printMsg(lvlError, format("path `%1%' is not in the Nix store") % path); - invalidatePath(path); - return; - } - - if (store.find(baseNameOf(path)) == store.end()) { - /* Check any referrers first. If we can invalidate them - first, then we can invalidate this path as well. */ - bool canInvalidate = true; - PathSet referrers; queryReferrers(path, referrers); - foreach (PathSet::iterator, i, referrers) - if (*i != path) { - verifyPath(*i, store, done, validPaths, repair, errors); - if (validPaths.find(*i) != validPaths.end()) - canInvalidate = false; - } - - if (canInvalidate) { - printMsg(lvlError, format("path `%1%' disappeared, removing from database...") % path); - invalidatePath(path); - } else { - printMsg(lvlError, format("path `%1%' disappeared, but it still has valid referrers!") % path); - if (repair) - try { - repairPath(path); - } catch (Error & e) { - printMsg(lvlError, format("warning: %1%") % e.msg()); - errors = true; - } - else errors = true; - } - - return; - } - - validPaths.insert(path); -} - - -bool LocalStore::pathContentsGood(const Path & path) -{ - std::map<Path, bool>::iterator i = pathContentsGoodCache.find(path); - if (i != pathContentsGoodCache.end()) return i->second; - printMsg(lvlInfo, format("checking path `%1%'...") % path); - ValidPathInfo info = queryPathInfo(path); - bool res; - if (!pathExists(path)) - res = false; - else { - HashResult current = hashPath(info.hash.type, path); - Hash nullHash(htSHA256); - res = info.hash == nullHash || info.hash == current.first; - } - pathContentsGoodCache[path] = res; - if (!res) printMsg(lvlError, format("path `%1%' is corrupted or missing!") % path); - return res; -} - - -void LocalStore::markContentsGood(const Path & path) -{ - pathContentsGoodCache[path] = true; -} - - -/* Functions for upgrading from the pre-SQLite database. */ - -PathSet LocalStore::queryValidPathsOld() -{ - PathSet paths; - Strings entries = readDirectory(settings.nixDBPath + "/info"); - foreach (Strings::iterator, i, entries) - if (i->at(0) != '.') paths.insert(settings.nixStore + "/" + *i); - return paths; -} - - -ValidPathInfo LocalStore::queryPathInfoOld(const Path & path) -{ - ValidPathInfo res; - res.path = path; - - /* Read the info file. */ - string baseName = baseNameOf(path); - Path infoFile = (format("%1%/info/%2%") % settings.nixDBPath % baseName).str(); - if (!pathExists(infoFile)) - throw Error(format("path `%1%' is not valid") % path); - string info = readFile(infoFile); - - /* Parse it. */ - Strings lines = tokenizeString<Strings>(info, "\n"); - - foreach (Strings::iterator, i, lines) { - string::size_type p = i->find(':'); - if (p == string::npos) - throw Error(format("corrupt line in `%1%': %2%") % infoFile % *i); - string name(*i, 0, p); - string value(*i, p + 2); - if (name == "References") { - Strings refs = tokenizeString<Strings>(value, " "); - res.references = PathSet(refs.begin(), refs.end()); - } else if (name == "Deriver") { - res.deriver = value; - } else if (name == "Hash") { - res.hash = parseHashField(path, value); - } else if (name == "Registered-At") { - int n = 0; - string2Int(value, n); - res.registrationTime = n; - } - } - - return res; -} - - -/* Upgrade from schema 5 (Nix 0.12) to schema 6 (Nix >= 0.15). */ -void LocalStore::upgradeStore6() -{ - printMsg(lvlError, "upgrading Nix store to new schema (this may take a while)..."); - - openDB(true); - - PathSet validPaths = queryValidPathsOld(); - - SQLiteTxn txn(db); - - foreach (PathSet::iterator, i, validPaths) { - addValidPath(queryPathInfoOld(*i), false); - std::cerr << "."; - } - - std::cerr << "|"; - - foreach (PathSet::iterator, i, validPaths) { - ValidPathInfo info = queryPathInfoOld(*i); - unsigned long long referrer = queryValidPathId(*i); - foreach (PathSet::iterator, j, info.references) - addReference(referrer, queryValidPathId(*j)); - std::cerr << "."; - } - - std::cerr << "\n"; - - txn.commit(); -} - - -#if defined(FS_IOC_SETFLAGS) && defined(FS_IOC_GETFLAGS) && defined(FS_IMMUTABLE_FL) - -static void makeMutable(const Path & path) -{ - checkInterrupt(); - - struct stat st = lstat(path); - - if (!S_ISDIR(st.st_mode) && !S_ISREG(st.st_mode)) return; - - if (S_ISDIR(st.st_mode)) { - Strings names = readDirectory(path); - foreach (Strings::iterator, i, names) - makeMutable(path + "/" + *i); - } - - /* The O_NOFOLLOW is important to prevent us from changing the - mutable bit on the target of a symlink (which would be a - security hole). */ - AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_NOFOLLOW); - if (fd == -1) { - if (errno == ELOOP) return; // it's a symlink - throw SysError(format("opening file `%1%'") % path); - } - - unsigned int flags = 0, old; - - /* Silently ignore errors getting/setting the immutable flag so - that we work correctly on filesystems that don't support it. */ - if (ioctl(fd, FS_IOC_GETFLAGS, &flags)) return; - old = flags; - flags &= ~FS_IMMUTABLE_FL; - if (old == flags) return; - if (ioctl(fd, FS_IOC_SETFLAGS, &flags)) return; -} - -/* Upgrade from schema 6 (Nix 0.15) to schema 7 (Nix >= 1.3). */ -void LocalStore::upgradeStore7() -{ - if (getuid() != 0) return; - printMsg(lvlError, "removing immutable bits from the Nix store (this may take a while)..."); - makeMutable(settings.nixStore); -} - -#else - -void LocalStore::upgradeStore7() -{ -} - -#endif - - -void LocalStore::vacuumDB() -{ - if (sqlite3_exec(db, "vacuum;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "vacuuming SQLite database"); -} - - -} diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh deleted file mode 100644 index 09639e74cf..0000000000 --- a/src/libstore/local-store.hh +++ /dev/null @@ -1,333 +0,0 @@ -#pragma once - -#include <string> - -#include "store-api.hh" -#include "util.hh" -#include "pathlocks.hh" - - -class sqlite3; -class sqlite3_stmt; - - -namespace nix { - - -/* Nix store and database schema version. Version 1 (or 0) was Nix <= - 0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10. - Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.16. Version 6 is - Nix 1.0. Version 7 is Nix 1.3. */ -const int nixSchemaVersion = 7; - - -extern string drvsLogDir; - - -struct Derivation; - - -struct OptimiseStats -{ - unsigned long totalFiles; - unsigned long sameContents; - unsigned long filesLinked; - unsigned long long bytesFreed; - unsigned long long blocksFreed; - OptimiseStats() - { - totalFiles = sameContents = filesLinked = 0; - bytesFreed = blocksFreed = 0; - } -}; - - -struct RunningSubstituter -{ - Path program; - Pid pid; - AutoCloseFD to, from, error; - FdSource fromBuf; - bool disabled; - RunningSubstituter() : disabled(false) { }; -}; - - -/* Wrapper object to close the SQLite database automatically. */ -struct SQLite -{ - sqlite3 * db; - SQLite() { db = 0; } - ~SQLite(); - operator sqlite3 * () { return db; } -}; - - -/* Wrapper object to create and destroy SQLite prepared statements. */ -struct SQLiteStmt -{ - sqlite3 * db; - sqlite3_stmt * stmt; - unsigned int curArg; - SQLiteStmt() { stmt = 0; } - void create(sqlite3 * db, const string & s); - void reset(); - ~SQLiteStmt(); - operator sqlite3_stmt * () { return stmt; } - void bind(const string & value); - void bind(int value); - void bind64(long long value); - void bind(); -}; - - -class LocalStore : public StoreAPI -{ -private: - typedef std::map<Path, RunningSubstituter> RunningSubstituters; - RunningSubstituters runningSubstituters; - - Path linksDir; - -public: - - /* Initialise the local store, upgrading the schema if - necessary. */ - LocalStore(bool reserveSpace = true); - - ~LocalStore(); - - /* Implementations of abstract store API methods. */ - - bool isValidPath(const Path & path); - - PathSet queryValidPaths(const PathSet & paths); - - PathSet queryAllValidPaths(); - - ValidPathInfo queryPathInfo(const Path & path); - - Hash queryPathHash(const Path & path); - - void queryReferences(const Path & path, PathSet & references); - - void queryReferrers(const Path & path, PathSet & referrers); - - Path queryDeriver(const Path & path); - - PathSet queryValidDerivers(const Path & path); - - PathSet queryDerivationOutputs(const Path & path); - - StringSet queryDerivationOutputNames(const Path & path); - - Path queryPathFromHashPart(const string & hashPart); - - PathSet querySubstitutablePaths(const PathSet & paths); - - void querySubstitutablePathInfos(const Path & substituter, - PathSet & paths, SubstitutablePathInfos & infos); - - void querySubstitutablePathInfos(const PathSet & paths, - SubstitutablePathInfos & infos); - - Path addToStore(const Path & srcPath, - bool recursive = true, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, bool repair = false); - - /* Like addToStore(), but the contents of the path are contained - in `dump', which is either a NAR serialisation (if recursive == - true) or simply the contents of a regular file (if recursive == - false). */ - Path addToStoreFromDump(const string & dump, const string & name, - bool recursive = true, HashType hashAlgo = htSHA256, bool repair = false); - - Path addTextToStore(const string & name, const string & s, - const PathSet & references, bool repair = false); - - void exportPath(const Path & path, bool sign, - Sink & sink); - - Paths importPaths(bool requireSignature, Source & source); - - void buildPaths(const PathSet & paths, BuildMode buildMode); - - void ensurePath(const Path & path); - - void addTempRoot(const Path & path); - - void addIndirectRoot(const Path & path); - - void syncWithGC(); - - Roots findRoots(); - - void collectGarbage(const GCOptions & options, GCResults & results); - - /* Optimise the disk space usage of the Nix store by hard-linking - files with the same contents. */ - void optimiseStore(OptimiseStats & stats); - - /* Optimise a single store path. */ - void optimisePath(const Path & path); - - /* Check the integrity of the Nix store. Returns true if errors - remain. */ - bool verifyStore(bool checkContents, bool repair); - - /* Register the validity of a path, i.e., that `path' exists, that - the paths referenced by it exists, and in the case of an output - path of a derivation, that it has been produced by a successful - execution of the derivation (or something equivalent). Also - register the hash of the file system contents of the path. The - hash must be a SHA-256 hash. */ - void registerValidPath(const ValidPathInfo & info); - - void registerValidPaths(const ValidPathInfos & infos); - - /* Register that the build of a derivation with output `path' has - failed. */ - void registerFailedPath(const Path & path); - - /* Query whether `path' previously failed to build. */ - bool hasPathFailed(const Path & path); - - PathSet queryFailedPaths(); - - void clearFailedPaths(const PathSet & paths); - - void vacuumDB(); - - /* Repair the contents of the given path by redownloading it using - a substituter (if available). */ - void repairPath(const Path & path); - - /* Check whether the given valid path exists and has the right - contents. */ - bool pathContentsGood(const Path & path); - - void markContentsGood(const Path & path); - - void setSubstituterEnv(); - -private: - - Path schemaPath; - - /* Lock file used for upgrading. */ - AutoCloseFD globalLock; - - /* The SQLite database object. */ - SQLite db; - - /* Some precompiled SQLite statements. */ - SQLiteStmt stmtRegisterValidPath; - SQLiteStmt stmtUpdatePathInfo; - SQLiteStmt stmtAddReference; - SQLiteStmt stmtQueryPathInfo; - SQLiteStmt stmtQueryReferences; - SQLiteStmt stmtQueryReferrers; - SQLiteStmt stmtInvalidatePath; - SQLiteStmt stmtRegisterFailedPath; - SQLiteStmt stmtHasPathFailed; - SQLiteStmt stmtQueryFailedPaths; - SQLiteStmt stmtClearFailedPath; - SQLiteStmt stmtAddDerivationOutput; - SQLiteStmt stmtQueryValidDerivers; - SQLiteStmt stmtQueryDerivationOutputs; - SQLiteStmt stmtQueryPathFromHashPart; - - /* Cache for pathContentsGood(). */ - std::map<Path, bool> pathContentsGoodCache; - - bool didSetSubstituterEnv; - - int getSchema(); - - void openDB(bool create); - - void makeStoreWritable(); - - unsigned long long queryValidPathId(const Path & path); - - unsigned long long addValidPath(const ValidPathInfo & info, bool checkOutputs = true); - - void addReference(unsigned long long referrer, unsigned long long reference); - - void appendReferrer(const Path & from, const Path & to, bool lock); - - void rewriteReferrers(const Path & path, bool purge, PathSet referrers); - - void invalidatePath(const Path & path); - - /* Delete a path from the Nix store. */ - void invalidatePathChecked(const Path & path); - - void verifyPath(const Path & path, const PathSet & store, - PathSet & done, PathSet & validPaths, bool repair, bool & errors); - - void updatePathInfo(const ValidPathInfo & info); - - void upgradeStore6(); - void upgradeStore7(); - PathSet queryValidPathsOld(); - ValidPathInfo queryPathInfoOld(const Path & path); - - struct GCState; - - void deleteGarbage(GCState & state, const Path & path); - - void tryToDelete(GCState & state, const Path & path); - - bool canReachRoot(GCState & state, PathSet & visited, const Path & path); - - void deletePathRecursive(GCState & state, const Path & path); - - bool isActiveTempFile(const GCState & state, - const Path & path, const string & suffix); - - int openGCLock(LockType lockType); - - void removeUnusedLinks(const GCState & state); - - void startSubstituter(const Path & substituter, - RunningSubstituter & runningSubstituter); - - string getLineFromSubstituter(RunningSubstituter & run); - - template<class T> T getIntLineFromSubstituter(RunningSubstituter & run); - - Path createTempDirInStore(); - - Path importPath(bool requireSignature, Source & source); - - void checkDerivationOutputs(const Path & drvPath, const Derivation & drv); - - void optimisePath_(OptimiseStats & stats, const Path & path); - - // Internal versions that are not wrapped in retry_sqlite. - bool isValidPath_(const Path & path); - void queryReferrers_(const Path & path, PathSet & referrers); -}; - - -typedef std::pair<dev_t, ino_t> Inode; -typedef set<Inode> InodesSeen; - - -/* "Fix", or canonicalise, the meta-data of the files in a store path - after it has been built. In particular: - - the last modification date on each file is set to 1 (i.e., - 00:00:01 1/1/1970 UTC) - - the permissions are set of 444 or 555 (i.e., read-only with or - without execute permission; setuid bits etc. are cleared) - - the owner and group are set to the Nix user and group, if we're - running as root. */ -void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & inodesSeen); -void canonicalisePathMetaData(const Path & path, uid_t fromUid); - -void canonicaliseTimestampAndPermissions(const Path & path); - -MakeError(PathInUse, Error); - -} diff --git a/src/libstore/local.mk b/src/libstore/local.mk deleted file mode 100644 index 2dddce740d..0000000000 --- a/src/libstore/local.mk +++ /dev/null @@ -1,28 +0,0 @@ -libraries += libstore - -libstore_NAME = libnixstore - -libstore_DIR := $(d) - -libstore_SOURCES := $(wildcard $(d)/*.cc) - -libstore_LIBS = libutil libformat - -libstore_LDFLAGS = -lsqlite3 -lbz2 - -libstore_CXXFLAGS = \ - -DNIX_STORE_DIR=\"$(storedir)\" \ - -DNIX_DATA_DIR=\"$(datadir)\" \ - -DNIX_STATE_DIR=\"$(localstatedir)/nix\" \ - -DNIX_LOG_DIR=\"$(localstatedir)/log/nix\" \ - -DNIX_CONF_DIR=\"$(sysconfdir)/nix\" \ - -DNIX_LIBEXEC_DIR=\"$(libexecdir)\" \ - -DNIX_BIN_DIR=\"$(bindir)\" \ - -DPACKAGE_VERSION=\"$(PACKAGE_VERSION)\" - -$(d)/local-store.cc: $(d)/schema.sql.hh - -%.sql.hh: %.sql - $(trace-gen) sed -e 's/"/\\"/g' -e 's/\(.*\)/"\1\\n"/' < $< > $@ || (rm $@ && exit 1) - -clean-files += $(d)/schema.sql.hh diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc deleted file mode 100644 index 1bf3f93782..0000000000 --- a/src/libstore/misc.cc +++ /dev/null @@ -1,220 +0,0 @@ -#include "misc.hh" -#include "store-api.hh" -#include "local-store.hh" -#include "globals.hh" - - -namespace nix { - - -Derivation derivationFromPath(StoreAPI & store, const Path & drvPath) -{ - assertStorePath(drvPath); - store.ensurePath(drvPath); - return parseDerivation(readFile(drvPath)); -} - - -void computeFSClosure(StoreAPI & store, const Path & path, - PathSet & paths, bool flipDirection, bool includeOutputs, bool includeDerivers) -{ - if (paths.find(path) != paths.end()) return; - paths.insert(path); - - PathSet edges; - - if (flipDirection) { - store.queryReferrers(path, edges); - - if (includeOutputs) { - PathSet derivers = store.queryValidDerivers(path); - foreach (PathSet::iterator, i, derivers) - edges.insert(*i); - } - - if (includeDerivers && isDerivation(path)) { - PathSet outputs = store.queryDerivationOutputs(path); - foreach (PathSet::iterator, i, outputs) - if (store.isValidPath(*i) && store.queryDeriver(*i) == path) - edges.insert(*i); - } - - } else { - store.queryReferences(path, edges); - - if (includeOutputs && isDerivation(path)) { - PathSet outputs = store.queryDerivationOutputs(path); - foreach (PathSet::iterator, i, outputs) - if (store.isValidPath(*i)) edges.insert(*i); - } - - if (includeDerivers) { - Path deriver = store.queryDeriver(path); - if (store.isValidPath(deriver)) edges.insert(deriver); - } - } - - foreach (PathSet::iterator, i, edges) - computeFSClosure(store, *i, paths, flipDirection, includeOutputs, includeDerivers); -} - - -Path findOutput(const Derivation & drv, string id) -{ - foreach (DerivationOutputs::const_iterator, i, drv.outputs) - if (i->first == id) return i->second.path; - throw Error(format("derivation has no output `%1%'") % id); -} - - -void queryMissing(StoreAPI & store, const PathSet & targets, - PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, - unsigned long long & downloadSize, unsigned long long & narSize) -{ - downloadSize = narSize = 0; - - PathSet todo(targets.begin(), targets.end()), done; - - /* Getting substitute info has high latency when using the binary - cache substituter. Thus it's essential to do substitute - queries in parallel as much as possible. To accomplish this - we do the following: - - - For all paths still to be processed (âtodoâ), we add all - paths for which we need info to the set âqueryâ. For an - unbuilt derivation this is the output paths; otherwise, it's - the path itself. - - - We get info about all paths in âqueryâ in parallel. - - - We process the results and add new items to âtodoâ if - necessary. E.g. if a path is substitutable, then we need to - get info on its references. - - - Repeat until âtodoâ is empty. - */ - - while (!todo.empty()) { - - PathSet query, todoDrv, todoNonDrv; - - foreach (PathSet::iterator, i, todo) { - if (done.find(*i) != done.end()) continue; - done.insert(*i); - - DrvPathWithOutputs i2 = parseDrvPathWithOutputs(*i); - - if (isDerivation(i2.first)) { - if (!store.isValidPath(i2.first)) { - // FIXME: we could try to substitute p. - unknown.insert(*i); - continue; - } - Derivation drv = derivationFromPath(store, i2.first); - - PathSet invalid; - foreach (DerivationOutputs::iterator, j, drv.outputs) - if (wantOutput(j->first, i2.second) - && !store.isValidPath(j->second.path)) - invalid.insert(j->second.path); - if (invalid.empty()) continue; - - todoDrv.insert(*i); - if (settings.useSubstitutes && !willBuildLocally(drv)) - query.insert(invalid.begin(), invalid.end()); - } - - else { - if (store.isValidPath(*i)) continue; - query.insert(*i); - todoNonDrv.insert(*i); - } - } - - todo.clear(); - - SubstitutablePathInfos infos; - store.querySubstitutablePathInfos(query, infos); - - foreach (PathSet::iterator, i, todoDrv) { - DrvPathWithOutputs i2 = parseDrvPathWithOutputs(*i); - - // FIXME: cache this - Derivation drv = derivationFromPath(store, i2.first); - - PathSet outputs; - bool mustBuild = false; - if (settings.useSubstitutes && !willBuildLocally(drv)) { - foreach (DerivationOutputs::iterator, j, drv.outputs) { - if (!wantOutput(j->first, i2.second)) continue; - if (!store.isValidPath(j->second.path)) { - if (infos.find(j->second.path) == infos.end()) - mustBuild = true; - else - outputs.insert(j->second.path); - } - } - } else - mustBuild = true; - - if (mustBuild) { - willBuild.insert(i2.first); - todo.insert(drv.inputSrcs.begin(), drv.inputSrcs.end()); - foreach (DerivationInputs::iterator, j, drv.inputDrvs) - todo.insert(makeDrvPathWithOutputs(j->first, j->second)); - } else - todoNonDrv.insert(outputs.begin(), outputs.end()); - } - - foreach (PathSet::iterator, i, todoNonDrv) { - done.insert(*i); - SubstitutablePathInfos::iterator info = infos.find(*i); - if (info != infos.end()) { - willSubstitute.insert(*i); - downloadSize += info->second.downloadSize; - narSize += info->second.narSize; - todo.insert(info->second.references.begin(), info->second.references.end()); - } else - unknown.insert(*i); - } - } -} - - -static void dfsVisit(StoreAPI & store, const PathSet & paths, - const Path & path, PathSet & visited, Paths & sorted, - PathSet & parents) -{ - if (parents.find(path) != parents.end()) - throw BuildError(format("cycle detected in the references of `%1%'") % path); - - if (visited.find(path) != visited.end()) return; - visited.insert(path); - parents.insert(path); - - PathSet references; - if (store.isValidPath(path)) - store.queryReferences(path, references); - - foreach (PathSet::iterator, i, references) - /* Don't traverse into paths that don't exist. That can - happen due to substitutes for non-existent paths. */ - if (*i != path && paths.find(*i) != paths.end()) - dfsVisit(store, paths, *i, visited, sorted, parents); - - sorted.push_front(path); - parents.erase(path); -} - - -Paths topoSortPaths(StoreAPI & store, const PathSet & paths) -{ - Paths sorted; - PathSet visited, parents; - foreach (PathSet::const_iterator, i, paths) - dfsVisit(store, paths, *i, visited, sorted, parents); - return sorted; -} - - -} diff --git a/src/libstore/misc.hh b/src/libstore/misc.hh deleted file mode 100644 index 144cb7f457..0000000000 --- a/src/libstore/misc.hh +++ /dev/null @@ -1,38 +0,0 @@ -#pragma once - -#include "derivations.hh" - - -namespace nix { - - -/* Read a derivation, after ensuring its existence through - ensurePath(). */ -Derivation derivationFromPath(StoreAPI & store, const Path & drvPath); - -/* Place in `paths' the set of all store paths in the file system - closure of `storePath'; that is, all paths than can be directly or - indirectly reached from it. `paths' is not cleared. If - `flipDirection' is true, the set of paths that can reach - `storePath' is returned; that is, the closures under the - `referrers' relation instead of the `references' relation is - returned. */ -void computeFSClosure(StoreAPI & store, const Path & path, - PathSet & paths, bool flipDirection = false, - bool includeOutputs = false, bool includeDerivers = false); - -/* Return the path corresponding to the output identifier `id' in the - given derivation. */ -Path findOutput(const Derivation & drv, string id); - -/* Given a set of paths that are to be built, return the set of - derivations that will be built, and the set of output paths that - will be substituted. */ -void queryMissing(StoreAPI & store, const PathSet & targets, - PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, - unsigned long long & downloadSize, unsigned long long & narSize); - -bool willBuildLocally(const Derivation & drv); - - -} diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc deleted file mode 100644 index d833f3aa05..0000000000 --- a/src/libstore/optimise-store.cc +++ /dev/null @@ -1,180 +0,0 @@ -#include "config.h" - -#include "util.hh" -#include "local-store.hh" -#include "globals.hh" - -#include <sys/types.h> -#include <sys/stat.h> -#include <unistd.h> -#include <errno.h> -#include <stdio.h> - - -namespace nix { - - -static void makeWritable(const Path & path) -{ - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path `%1%'") % path); - if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1) - throw SysError(format("changing writability of `%1%'") % path); -} - - -struct MakeReadOnly -{ - Path path; - MakeReadOnly(const Path & path) : path(path) { } - ~MakeReadOnly() - { - try { - /* This will make the path read-only. */ - if (path != "") canonicaliseTimestampAndPermissions(path); - } catch (...) { - ignoreException(); - } - } -}; - - -void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path) -{ - checkInterrupt(); - - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path `%1%'") % path); - - if (S_ISDIR(st.st_mode)) { - Strings names = readDirectory(path); - foreach (Strings::iterator, i, names) - optimisePath_(stats, path + "/" + *i); - return; - } - - /* We can hard link regular files and maybe symlinks. */ - if (!S_ISREG(st.st_mode) -#if CAN_LINK_SYMLINK - && !S_ISLNK(st.st_mode) -#endif - ) return; - - /* Sometimes SNAFUs can cause files in the Nix store to be - modified, in particular when running programs as root under - NixOS (example: $fontconfig/var/cache being modified). Skip - those files. FIXME: check the modification time. */ - if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) { - printMsg(lvlError, format("skipping suspicious writable file `%1%'") % path); - return; - } - - /* Hash the file. Note that hashPath() returns the hash over the - NAR serialisation, which includes the execute bit on the file. - Thus, executable and non-executable files with the same - contents *won't* be linked (which is good because otherwise the - permissions would be screwed up). - - Also note that if `path' is a symlink, then we're hashing the - contents of the symlink (i.e. the result of readlink()), not - the contents of the target (which may not even exist). */ - Hash hash = hashPath(htSHA256, path).first; - stats.totalFiles++; - printMsg(lvlDebug, format("`%1%' has hash `%2%'") % path % printHash(hash)); - - /* Check if this is a known hash. */ - Path linkPath = linksDir + "/" + printHash32(hash); - - if (!pathExists(linkPath)) { - /* Nope, create a hard link in the links directory. */ - if (link(path.c_str(), linkPath.c_str()) == 0) return; - if (errno != EEXIST) - throw SysError(format("cannot link `%1%' to `%2%'") % linkPath % path); - /* Fall through if another process created âlinkPathâ before - we did. */ - } - - /* Yes! We've seen a file with the same contents. Replace the - current file with a hard link to that file. */ - struct stat stLink; - if (lstat(linkPath.c_str(), &stLink)) - throw SysError(format("getting attributes of path `%1%'") % linkPath); - - stats.sameContents++; - if (st.st_ino == stLink.st_ino) { - printMsg(lvlDebug, format("`%1%' is already linked to `%2%'") % path % linkPath); - return; - } - - printMsg(lvlTalkative, format("linking `%1%' to `%2%'") % path % linkPath); - - /* Make the containing directory writable, but only if it's not - the store itself (we don't want or need to mess with its - permissions). */ - bool mustToggle = !isStorePath(path); - if (mustToggle) makeWritable(dirOf(path)); - - /* When we're done, make the directory read-only again and reset - its timestamp back to 0. */ - MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : ""); - - Path tempLink = (format("%1%/.tmp-link-%2%-%3%") - % settings.nixStore % getpid() % rand()).str(); - - if (link(linkPath.c_str(), tempLink.c_str()) == -1) { - if (errno == EMLINK) { - /* Too many links to the same file (>= 32000 on most file - systems). This is likely to happen with empty files. - Just shrug and ignore. */ - if (st.st_size) - printMsg(lvlInfo, format("`%1%' has maximum number of links") % linkPath); - return; - } - throw SysError(format("cannot link `%1%' to `%2%'") % tempLink % linkPath); - } - - /* Atomically replace the old file with the new hard link. */ - if (rename(tempLink.c_str(), path.c_str()) == -1) { - if (unlink(tempLink.c_str()) == -1) - printMsg(lvlError, format("unable to unlink `%1%'") % tempLink); - if (errno == EMLINK) { - /* Some filesystems generate too many links on the rename, - rather than on the original link. (Probably it - temporarily increases the st_nlink field before - decreasing it again.) */ - if (st.st_size) - printMsg(lvlInfo, format("`%1%' has maximum number of links") % linkPath); - return; - } - throw SysError(format("cannot rename `%1%' to `%2%'") % tempLink % path); - } - - stats.filesLinked++; - stats.bytesFreed += st.st_size; - stats.blocksFreed += st.st_blocks; -} - - -void LocalStore::optimiseStore(OptimiseStats & stats) -{ - PathSet paths = queryAllValidPaths(); - - foreach (PathSet::iterator, i, paths) { - addTempRoot(*i); - if (!isValidPath(*i)) continue; /* path was GC'ed, probably */ - startNest(nest, lvlChatty, format("hashing files in `%1%'") % *i); - optimisePath_(stats, *i); - } -} - - -void LocalStore::optimisePath(const Path & path) -{ - OptimiseStats stats; - if (settings.autoOptimiseStore) optimisePath_(stats, path); -} - - -} diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc deleted file mode 100644 index b858ed238d..0000000000 --- a/src/libstore/pathlocks.cc +++ /dev/null @@ -1,199 +0,0 @@ -#include "pathlocks.hh" -#include "util.hh" - -#include <cerrno> -#include <cstdlib> - -#include <sys/types.h> -#include <sys/stat.h> -#include <fcntl.h> - - -namespace nix { - - -int openLockFile(const Path & path, bool create) -{ - AutoCloseFD fd; - - fd = open(path.c_str(), O_RDWR | (create ? O_CREAT : 0), 0600); - if (fd == -1 && (create || errno != ENOENT)) - throw SysError(format("opening lock file `%1%'") % path); - - closeOnExec(fd); - - return fd.borrow(); -} - - -void deleteLockFile(const Path & path, int fd) -{ - /* Get rid of the lock file. Have to be careful not to introduce - races. Write a (meaningless) token to the file to indicate to - other processes waiting on this lock that the lock is stale - (deleted). */ - unlink(path.c_str()); - writeFull(fd, (const unsigned char *) "d", 1); - /* Note that the result of unlink() is ignored; removing the lock - file is an optimisation, not a necessity. */ -} - - -bool lockFile(int fd, LockType lockType, bool wait) -{ - struct flock lock; - if (lockType == ltRead) lock.l_type = F_RDLCK; - else if (lockType == ltWrite) lock.l_type = F_WRLCK; - else if (lockType == ltNone) lock.l_type = F_UNLCK; - else abort(); - lock.l_whence = SEEK_SET; - lock.l_start = 0; - lock.l_len = 0; /* entire file */ - - if (wait) { - while (fcntl(fd, F_SETLKW, &lock) != 0) { - checkInterrupt(); - if (errno != EINTR) - throw SysError(format("acquiring/releasing lock")); - } - } else { - while (fcntl(fd, F_SETLK, &lock) != 0) { - checkInterrupt(); - if (errno == EACCES || errno == EAGAIN) return false; - if (errno != EINTR) - throw SysError(format("acquiring/releasing lock")); - } - } - - return true; -} - - -/* This enables us to check whether are not already holding a lock on - a file ourselves. POSIX locks (fcntl) suck in this respect: if we - close a descriptor, the previous lock will be closed as well. And - there is no way to query whether we already have a lock (F_GETLK - only works on locks held by other processes). */ -static StringSet lockedPaths; /* !!! not thread-safe */ - - -PathLocks::PathLocks() - : deletePaths(false) -{ -} - - -PathLocks::PathLocks(const PathSet & paths, const string & waitMsg) - : deletePaths(false) -{ - lockPaths(paths, waitMsg); -} - - -bool PathLocks::lockPaths(const PathSet & _paths, - const string & waitMsg, bool wait) -{ - assert(fds.empty()); - - /* Note that `fds' is built incrementally so that the destructor - will only release those locks that we have already acquired. */ - - /* Sort the paths. This assures that locks are always acquired in - the same order, thus preventing deadlocks. */ - Paths paths(_paths.begin(), _paths.end()); - paths.sort(); - - /* Acquire the lock for each path. */ - foreach (Paths::iterator, i, paths) { - checkInterrupt(); - Path path = *i; - Path lockPath = path + ".lock"; - - debug(format("locking path `%1%'") % path); - - if (lockedPaths.find(lockPath) != lockedPaths.end()) - throw Error("deadlock: trying to re-acquire self-held lock"); - - AutoCloseFD fd; - - while (1) { - - /* Open/create the lock file. */ - fd = openLockFile(lockPath, true); - - /* Acquire an exclusive lock. */ - if (!lockFile(fd, ltWrite, false)) { - if (wait) { - if (waitMsg != "") printMsg(lvlError, waitMsg); - lockFile(fd, ltWrite, true); - } else { - /* Failed to lock this path; release all other - locks. */ - unlock(); - return false; - } - } - - debug(format("lock acquired on `%1%'") % lockPath); - - /* Check that the lock file hasn't become stale (i.e., - hasn't been unlinked). */ - struct stat st; - if (fstat(fd, &st) == -1) - throw SysError(format("statting lock file `%1%'") % lockPath); - if (st.st_size != 0) - /* This lock file has been unlinked, so we're holding - a lock on a deleted file. This means that other - processes may create and acquire a lock on - `lockPath', and proceed. So we must retry. */ - debug(format("open lock file `%1%' has become stale") % lockPath); - else - break; - } - - /* Use borrow so that the descriptor isn't closed. */ - fds.push_back(FDPair(fd.borrow(), lockPath)); - lockedPaths.insert(lockPath); - } - - return true; -} - - -PathLocks::~PathLocks() -{ - unlock(); -} - - -void PathLocks::unlock() -{ - foreach (list<FDPair>::iterator, i, fds) { - if (deletePaths) deleteLockFile(i->second, i->first); - - lockedPaths.erase(i->second); - if (close(i->first) == -1) - printMsg(lvlError, - format("error (ignored): cannot close lock file on `%1%'") % i->second); - - debug(format("lock released on `%1%'") % i->second); - } - - fds.clear(); -} - - -void PathLocks::setDeletion(bool deletePaths) -{ - this->deletePaths = deletePaths; -} - - -bool pathIsLockedByMe(const Path & path) -{ - Path lockPath = path + ".lock"; - return lockedPaths.find(lockPath) != lockedPaths.end(); -} - - -} diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh deleted file mode 100644 index 8a6b1450da..0000000000 --- a/src/libstore/pathlocks.hh +++ /dev/null @@ -1,45 +0,0 @@ -#pragma once - -#include "types.hh" - - -namespace nix { - - -/* Open (possibly create) a lock file and return the file descriptor. - -1 is returned if create is false and the lock could not be opened - because it doesn't exist. Any other error throws an exception. */ -int openLockFile(const Path & path, bool create); - -/* Delete an open lock file. */ -void deleteLockFile(const Path & path, int fd); - -enum LockType { ltRead, ltWrite, ltNone }; - -bool lockFile(int fd, LockType lockType, bool wait); - - -class PathLocks -{ -private: - typedef std::pair<int, Path> FDPair; - list<FDPair> fds; - bool deletePaths; - -public: - PathLocks(); - PathLocks(const PathSet & paths, - const string & waitMsg = ""); - bool lockPaths(const PathSet & _paths, - const string & waitMsg = "", - bool wait = true); - ~PathLocks(); - void unlock(); - void setDeletion(bool deletePaths); -}; - - -bool pathIsLockedByMe(const Path & path); - - -} diff --git a/src/libstore/references.cc b/src/libstore/references.cc deleted file mode 100644 index 282b848938..0000000000 --- a/src/libstore/references.cc +++ /dev/null @@ -1,122 +0,0 @@ -#include "references.hh" -#include "hash.hh" -#include "util.hh" -#include "archive.hh" - -#include <map> -#include <cstdlib> - - -namespace nix { - - -static unsigned int refLength = 32; /* characters */ - - -static void search(const unsigned char * s, unsigned int len, - StringSet & hashes, StringSet & seen) -{ - static bool initialised = false; - static bool isBase32[256]; - if (!initialised) { - for (unsigned int i = 0; i < 256; ++i) isBase32[i] = false; - for (unsigned int i = 0; i < base32Chars.size(); ++i) - isBase32[(unsigned char) base32Chars[i]] = true; - initialised = true; - } - - for (unsigned int i = 0; i + refLength <= len; ) { - int j; - bool match = true; - for (j = refLength - 1; j >= 0; --j) - if (!isBase32[(unsigned char) s[i + j]]) { - i += j + 1; - match = false; - break; - } - if (!match) continue; - string ref((const char *) s + i, refLength); - if (hashes.find(ref) != hashes.end()) { - debug(format("found reference to `%1%' at offset `%2%'") - % ref % i); - seen.insert(ref); - hashes.erase(ref); - } - ++i; - } -} - - -struct RefScanSink : Sink -{ - HashSink hashSink; - StringSet hashes; - StringSet seen; - - string tail; - - RefScanSink() : hashSink(htSHA256) { } - - void operator () (const unsigned char * data, size_t len); -}; - - -void RefScanSink::operator () (const unsigned char * data, size_t len) -{ - hashSink(data, len); - - /* It's possible that a reference spans the previous and current - fragment, so search in the concatenation of the tail of the - previous fragment and the start of the current fragment. */ - string s = tail + string((const char *) data, len > refLength ? refLength : len); - search((const unsigned char *) s.data(), s.size(), hashes, seen); - - search(data, len, hashes, seen); - - unsigned int tailLen = len <= refLength ? len : refLength; - tail = - string(tail, tail.size() < refLength - tailLen ? 0 : tail.size() - (refLength - tailLen)) + - string((const char *) data + len - tailLen, tailLen); -} - - -PathSet scanForReferences(const string & path, - const PathSet & refs, HashResult & hash) -{ - RefScanSink sink; - std::map<string, Path> backMap; - - /* For efficiency (and a higher hit rate), just search for the - hash part of the file name. (This assumes that all references - have the form `HASH-bla'). */ - foreach (PathSet::const_iterator, i, refs) { - string baseName = baseNameOf(*i); - string::size_type pos = baseName.find('-'); - if (pos == string::npos) - throw Error(format("bad reference `%1%'") % *i); - string s = string(baseName, 0, pos); - assert(s.size() == refLength); - assert(backMap.find(s) == backMap.end()); - // parseHash(htSHA256, s); - sink.hashes.insert(s); - backMap[s] = *i; - } - - /* Look for the hashes in the NAR dump of the path. */ - dumpPath(path, sink); - - /* Map the hashes found back to their store paths. */ - PathSet found; - foreach (StringSet::iterator, i, sink.seen) { - std::map<string, Path>::iterator j; - if ((j = backMap.find(*i)) == backMap.end()) abort(); - found.insert(j->second); - } - - hash = sink.hashSink.finish(); - - return found; -} - - -} diff --git a/src/libstore/references.hh b/src/libstore/references.hh deleted file mode 100644 index 013809d122..0000000000 --- a/src/libstore/references.hh +++ /dev/null @@ -1,11 +0,0 @@ -#pragma once - -#include "types.hh" -#include "hash.hh" - -namespace nix { - -PathSet scanForReferences(const Path & path, const PathSet & refs, - HashResult & hash); - -} diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc deleted file mode 100644 index 4619206932..0000000000 --- a/src/libstore/remote-store.cc +++ /dev/null @@ -1,602 +0,0 @@ -#include "serialise.hh" -#include "util.hh" -#include "remote-store.hh" -#include "worker-protocol.hh" -#include "archive.hh" -#include "affinity.hh" -#include "globals.hh" - -#include <sys/types.h> -#include <sys/stat.h> -#include <sys/socket.h> -#include <sys/un.h> -#include <fcntl.h> - -#include <iostream> -#include <unistd.h> -#include <cstring> - -namespace nix { - - -Path readStorePath(Source & from) -{ - Path path = readString(from); - assertStorePath(path); - return path; -} - - -template<class T> T readStorePaths(Source & from) -{ - T paths = readStrings<T>(from); - foreach (typename T::iterator, i, paths) assertStorePath(*i); - return paths; -} - -template PathSet readStorePaths(Source & from); - - -RemoteStore::RemoteStore() -{ - initialised = false; -} - - -void RemoteStore::openConnection(bool reserveSpace) -{ - if (initialised) return; - initialised = true; - - string remoteMode = getEnv("NIX_REMOTE"); - - if (remoteMode == "daemon") - /* Connect to a daemon that does the privileged work for - us. */ - connectToDaemon(); - else - throw Error(format("invalid setting for NIX_REMOTE, `%1%'") % remoteMode); - - from.fd = fdSocket; - to.fd = fdSocket; - - /* Send the magic greeting, check for the reply. */ - try { - writeInt(WORKER_MAGIC_1, to); - to.flush(); - unsigned int magic = readInt(from); - if (magic != WORKER_MAGIC_2) throw Error("protocol mismatch"); - - daemonVersion = readInt(from); - if (GET_PROTOCOL_MAJOR(daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION)) - throw Error("Nix daemon protocol version not supported"); - writeInt(PROTOCOL_VERSION, to); - - if (GET_PROTOCOL_MINOR(daemonVersion) >= 14) { - int cpu = settings.lockCPU ? lockToCurrentCPU() : -1; - if (cpu != -1) { - writeInt(1, to); - writeInt(cpu, to); - } else - writeInt(0, to); - } - - if (GET_PROTOCOL_MINOR(daemonVersion) >= 11) - writeInt(reserveSpace, to); - - processStderr(); - } - catch (Error & e) { - throw Error(format("cannot start worker (%1%)") - % e.msg()); - } - - setOptions(); -} - - -void RemoteStore::connectToDaemon() -{ - fdSocket = socket(PF_UNIX, SOCK_STREAM, 0); - if (fdSocket == -1) - throw SysError("cannot create Unix domain socket"); - closeOnExec(fdSocket); - - string socketPath = settings.nixDaemonSocketFile; - - /* Urgh, sockaddr_un allows path names of only 108 characters. So - chdir to the socket directory so that we can pass a relative - path name. !!! this is probably a bad idea in multi-threaded - applications... */ - AutoCloseFD fdPrevDir = open(".", O_RDONLY); - if (fdPrevDir == -1) throw SysError("couldn't open current directory"); - chdir(dirOf(socketPath).c_str()); - Path socketPathRel = "./" + baseNameOf(socketPath); - - struct sockaddr_un addr; - addr.sun_family = AF_UNIX; - if (socketPathRel.size() >= sizeof(addr.sun_path)) - throw Error(format("socket path `%1%' is too long") % socketPathRel); - using namespace std; - strcpy(addr.sun_path, socketPathRel.c_str()); - - if (connect(fdSocket, (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError(format("cannot connect to daemon at `%1%'") % socketPath); - - if (fchdir(fdPrevDir) == -1) - throw SysError("couldn't change back to previous directory"); -} - - -RemoteStore::~RemoteStore() -{ - try { - to.flush(); - fdSocket.close(); - if (child != -1) - child.wait(true); - } catch (...) { - ignoreException(); - } -} - - -void RemoteStore::setOptions() -{ - writeInt(wopSetOptions, to); - - writeInt(settings.keepFailed, to); - writeInt(settings.keepGoing, to); - writeInt(settings.tryFallback, to); - writeInt(verbosity, to); - writeInt(settings.maxBuildJobs, to); - writeInt(settings.maxSilentTime, to); - if (GET_PROTOCOL_MINOR(daemonVersion) >= 2) - writeInt(settings.useBuildHook, to); - if (GET_PROTOCOL_MINOR(daemonVersion) >= 4) { - writeInt(settings.buildVerbosity, to); - writeInt(logType, to); - writeInt(settings.printBuildTrace, to); - } - if (GET_PROTOCOL_MINOR(daemonVersion) >= 6) - writeInt(settings.buildCores, to); - if (GET_PROTOCOL_MINOR(daemonVersion) >= 10) - writeInt(settings.useSubstitutes, to); - - if (GET_PROTOCOL_MINOR(daemonVersion) >= 12) { - Settings::SettingsMap overrides = settings.getOverrides(); - writeInt(overrides.size(), to); - foreach (Settings::SettingsMap::iterator, i, overrides) { - writeString(i->first, to); - writeString(i->second, to); - } - } - - processStderr(); -} - - -bool RemoteStore::isValidPath(const Path & path) -{ - openConnection(); - writeInt(wopIsValidPath, to); - writeString(path, to); - processStderr(); - unsigned int reply = readInt(from); - return reply != 0; -} - - -PathSet RemoteStore::queryValidPaths(const PathSet & paths) -{ - openConnection(); - if (GET_PROTOCOL_MINOR(daemonVersion) < 12) { - PathSet res; - foreach (PathSet::const_iterator, i, paths) - if (isValidPath(*i)) res.insert(*i); - return res; - } else { - writeInt(wopQueryValidPaths, to); - writeStrings(paths, to); - processStderr(); - return readStorePaths<PathSet>(from); - } -} - - -PathSet RemoteStore::queryAllValidPaths() -{ - openConnection(); - writeInt(wopQueryAllValidPaths, to); - processStderr(); - return readStorePaths<PathSet>(from); -} - - -PathSet RemoteStore::querySubstitutablePaths(const PathSet & paths) -{ - openConnection(); - if (GET_PROTOCOL_MINOR(daemonVersion) < 12) { - PathSet res; - foreach (PathSet::const_iterator, i, paths) { - writeInt(wopHasSubstitutes, to); - writeString(*i, to); - processStderr(); - if (readInt(from)) res.insert(*i); - } - return res; - } else { - writeInt(wopQuerySubstitutablePaths, to); - writeStrings(paths, to); - processStderr(); - return readStorePaths<PathSet>(from); - } -} - - -void RemoteStore::querySubstitutablePathInfos(const PathSet & paths, - SubstitutablePathInfos & infos) -{ - if (paths.empty()) return; - - openConnection(); - - if (GET_PROTOCOL_MINOR(daemonVersion) < 3) return; - - if (GET_PROTOCOL_MINOR(daemonVersion) < 12) { - - foreach (PathSet::const_iterator, i, paths) { - SubstitutablePathInfo info; - writeInt(wopQuerySubstitutablePathInfo, to); - writeString(*i, to); - processStderr(); - unsigned int reply = readInt(from); - if (reply == 0) continue; - info.deriver = readString(from); - if (info.deriver != "") assertStorePath(info.deriver); - info.references = readStorePaths<PathSet>(from); - info.downloadSize = readLongLong(from); - info.narSize = GET_PROTOCOL_MINOR(daemonVersion) >= 7 ? readLongLong(from) : 0; - infos[*i] = info; - } - - } else { - - writeInt(wopQuerySubstitutablePathInfos, to); - writeStrings(paths, to); - processStderr(); - unsigned int count = readInt(from); - for (unsigned int n = 0; n < count; n++) { - Path path = readStorePath(from); - SubstitutablePathInfo & info(infos[path]); - info.deriver = readString(from); - if (info.deriver != "") assertStorePath(info.deriver); - info.references = readStorePaths<PathSet>(from); - info.downloadSize = readLongLong(from); - info.narSize = readLongLong(from); - } - - } -} - - -ValidPathInfo RemoteStore::queryPathInfo(const Path & path) -{ - openConnection(); - writeInt(wopQueryPathInfo, to); - writeString(path, to); - processStderr(); - ValidPathInfo info; - info.path = path; - info.deriver = readString(from); - if (info.deriver != "") assertStorePath(info.deriver); - info.hash = parseHash(htSHA256, readString(from)); - info.references = readStorePaths<PathSet>(from); - info.registrationTime = readInt(from); - info.narSize = readLongLong(from); - return info; -} - - -Hash RemoteStore::queryPathHash(const Path & path) -{ - openConnection(); - writeInt(wopQueryPathHash, to); - writeString(path, to); - processStderr(); - string hash = readString(from); - return parseHash(htSHA256, hash); -} - - -void RemoteStore::queryReferences(const Path & path, - PathSet & references) -{ - openConnection(); - writeInt(wopQueryReferences, to); - writeString(path, to); - processStderr(); - PathSet references2 = readStorePaths<PathSet>(from); - references.insert(references2.begin(), references2.end()); -} - - -void RemoteStore::queryReferrers(const Path & path, - PathSet & referrers) -{ - openConnection(); - writeInt(wopQueryReferrers, to); - writeString(path, to); - processStderr(); - PathSet referrers2 = readStorePaths<PathSet>(from); - referrers.insert(referrers2.begin(), referrers2.end()); -} - - -Path RemoteStore::queryDeriver(const Path & path) -{ - openConnection(); - writeInt(wopQueryDeriver, to); - writeString(path, to); - processStderr(); - Path drvPath = readString(from); - if (drvPath != "") assertStorePath(drvPath); - return drvPath; -} - - -PathSet RemoteStore::queryValidDerivers(const Path & path) -{ - openConnection(); - writeInt(wopQueryValidDerivers, to); - writeString(path, to); - processStderr(); - return readStorePaths<PathSet>(from); -} - - -PathSet RemoteStore::queryDerivationOutputs(const Path & path) -{ - openConnection(); - writeInt(wopQueryDerivationOutputs, to); - writeString(path, to); - processStderr(); - return readStorePaths<PathSet>(from); -} - - -PathSet RemoteStore::queryDerivationOutputNames(const Path & path) -{ - openConnection(); - writeInt(wopQueryDerivationOutputNames, to); - writeString(path, to); - processStderr(); - return readStrings<PathSet>(from); -} - - -Path RemoteStore::queryPathFromHashPart(const string & hashPart) -{ - openConnection(); - writeInt(wopQueryPathFromHashPart, to); - writeString(hashPart, to); - processStderr(); - Path path = readString(from); - if (!path.empty()) assertStorePath(path); - return path; -} - - -Path RemoteStore::addToStore(const Path & _srcPath, - bool recursive, HashType hashAlgo, PathFilter & filter, bool repair) -{ - if (repair) throw Error("repairing is not supported when building through the Nix daemon"); - - openConnection(); - - Path srcPath(absPath(_srcPath)); - - writeInt(wopAddToStore, to); - writeString(baseNameOf(srcPath), to); - /* backwards compatibility hack */ - writeInt((hashAlgo == htSHA256 && recursive) ? 0 : 1, to); - writeInt(recursive ? 1 : 0, to); - writeString(printHashType(hashAlgo), to); - dumpPath(srcPath, to, filter); - processStderr(); - return readStorePath(from); -} - - -Path RemoteStore::addTextToStore(const string & name, const string & s, - const PathSet & references, bool repair) -{ - if (repair) throw Error("repairing is not supported when building through the Nix daemon"); - - openConnection(); - writeInt(wopAddTextToStore, to); - writeString(name, to); - writeString(s, to); - writeStrings(references, to); - - processStderr(); - return readStorePath(from); -} - - -void RemoteStore::exportPath(const Path & path, bool sign, - Sink & sink) -{ - openConnection(); - writeInt(wopExportPath, to); - writeString(path, to); - writeInt(sign ? 1 : 0, to); - processStderr(&sink); /* sink receives the actual data */ - readInt(from); -} - - -Paths RemoteStore::importPaths(bool requireSignature, Source & source) -{ - openConnection(); - writeInt(wopImportPaths, to); - /* We ignore requireSignature, since the worker forces it to true - anyway. */ - processStderr(0, &source); - return readStorePaths<Paths>(from); -} - - -void RemoteStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode) -{ - if (buildMode != bmNormal) throw Error("repairing or checking is not supported when building through the Nix daemon"); - openConnection(); - writeInt(wopBuildPaths, to); - if (GET_PROTOCOL_MINOR(daemonVersion) >= 13) - writeStrings(drvPaths, to); - else { - /* For backwards compatibility with old daemons, strip output - identifiers. */ - PathSet drvPaths2; - foreach (PathSet::const_iterator, i, drvPaths) - drvPaths2.insert(string(*i, 0, i->find('!'))); - writeStrings(drvPaths2, to); - } - processStderr(); - readInt(from); -} - - -void RemoteStore::ensurePath(const Path & path) -{ - openConnection(); - writeInt(wopEnsurePath, to); - writeString(path, to); - processStderr(); - readInt(from); -} - - -void RemoteStore::addTempRoot(const Path & path) -{ - openConnection(); - writeInt(wopAddTempRoot, to); - writeString(path, to); - processStderr(); - readInt(from); -} - - -void RemoteStore::addIndirectRoot(const Path & path) -{ - openConnection(); - writeInt(wopAddIndirectRoot, to); - writeString(path, to); - processStderr(); - readInt(from); -} - - -void RemoteStore::syncWithGC() -{ - openConnection(); - writeInt(wopSyncWithGC, to); - processStderr(); - readInt(from); -} - - -Roots RemoteStore::findRoots() -{ - openConnection(); - writeInt(wopFindRoots, to); - processStderr(); - unsigned int count = readInt(from); - Roots result; - while (count--) { - Path link = readString(from); - Path target = readStorePath(from); - result[link] = target; - } - return result; -} - - -void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) -{ - openConnection(false); - - writeInt(wopCollectGarbage, to); - writeInt(options.action, to); - writeStrings(options.pathsToDelete, to); - writeInt(options.ignoreLiveness, to); - writeLongLong(options.maxFreed, to); - writeInt(0, to); - if (GET_PROTOCOL_MINOR(daemonVersion) >= 5) { - /* removed options */ - writeInt(0, to); - writeInt(0, to); - } - - processStderr(); - - results.paths = readStrings<PathSet>(from); - results.bytesFreed = readLongLong(from); - readLongLong(from); // obsolete -} - - -PathSet RemoteStore::queryFailedPaths() -{ - openConnection(); - writeInt(wopQueryFailedPaths, to); - processStderr(); - return readStorePaths<PathSet>(from); -} - - -void RemoteStore::clearFailedPaths(const PathSet & paths) -{ - openConnection(); - writeInt(wopClearFailedPaths, to); - writeStrings(paths, to); - processStderr(); - readInt(from); -} - - -void RemoteStore::processStderr(Sink * sink, Source * source) -{ - to.flush(); - unsigned int msg; - while ((msg = readInt(from)) == STDERR_NEXT - || msg == STDERR_READ || msg == STDERR_WRITE) { - if (msg == STDERR_WRITE) { - string s = readString(from); - if (!sink) throw Error("no sink"); - (*sink)((const unsigned char *) s.data(), s.size()); - } - else if (msg == STDERR_READ) { - if (!source) throw Error("no source"); - size_t len = readInt(from); - unsigned char * buf = new unsigned char[len]; - AutoDeleteArray<unsigned char> d(buf); - writeString(buf, source->read(buf, len), to); - to.flush(); - } - else { - string s = readString(from); - writeToStderr(s); - } - } - if (msg == STDERR_ERROR) { - string error = readString(from); - unsigned int status = GET_PROTOCOL_MINOR(daemonVersion) >= 8 ? readInt(from) : 1; - throw Error(format("%1%") % error, status); - } - else if (msg != STDERR_LAST) - throw Error("protocol error processing standard error"); -} - - -} diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh deleted file mode 100644 index 04b60fce4b..0000000000 --- a/src/libstore/remote-store.hh +++ /dev/null @@ -1,104 +0,0 @@ -#pragma once - -#include <string> - -#include "store-api.hh" - - -namespace nix { - - -class Pipe; -class Pid; -struct FdSink; -struct FdSource; - - -class RemoteStore : public StoreAPI -{ -public: - - RemoteStore(); - - ~RemoteStore(); - - /* Implementations of abstract store API methods. */ - - bool isValidPath(const Path & path); - - PathSet queryValidPaths(const PathSet & paths); - - PathSet queryAllValidPaths(); - - ValidPathInfo queryPathInfo(const Path & path); - - Hash queryPathHash(const Path & path); - - void queryReferences(const Path & path, PathSet & references); - - void queryReferrers(const Path & path, PathSet & referrers); - - Path queryDeriver(const Path & path); - - PathSet queryValidDerivers(const Path & path); - - PathSet queryDerivationOutputs(const Path & path); - - StringSet queryDerivationOutputNames(const Path & path); - - Path queryPathFromHashPart(const string & hashPart); - - PathSet querySubstitutablePaths(const PathSet & paths); - - void querySubstitutablePathInfos(const PathSet & paths, - SubstitutablePathInfos & infos); - - Path addToStore(const Path & srcPath, - bool recursive = true, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, bool repair = false); - - Path addTextToStore(const string & name, const string & s, - const PathSet & references, bool repair = false); - - void exportPath(const Path & path, bool sign, - Sink & sink); - - Paths importPaths(bool requireSignature, Source & source); - - void buildPaths(const PathSet & paths, BuildMode buildMode); - - void ensurePath(const Path & path); - - void addTempRoot(const Path & path); - - void addIndirectRoot(const Path & path); - - void syncWithGC(); - - Roots findRoots(); - - void collectGarbage(const GCOptions & options, GCResults & results); - - PathSet queryFailedPaths(); - - void clearFailedPaths(const PathSet & paths); - -private: - AutoCloseFD fdSocket; - FdSink to; - FdSource from; - Pid child; - unsigned int daemonVersion; - bool initialised; - - void openConnection(bool reserveSpace = true); - - void processStderr(Sink * sink = 0, Source * source = 0); - - void connectToDaemon(); - - void setOptions(); -}; - - -} diff --git a/src/libstore/schema.sql b/src/libstore/schema.sql deleted file mode 100644 index c1b4a689af..0000000000 --- a/src/libstore/schema.sql +++ /dev/null @@ -1,44 +0,0 @@ -create table if not exists ValidPaths ( - id integer primary key autoincrement not null, - path text unique not null, - hash text not null, - registrationTime integer not null, - deriver text, - narSize integer -); - -create table if not exists Refs ( - referrer integer not null, - reference integer not null, - primary key (referrer, reference), - foreign key (referrer) references ValidPaths(id) on delete cascade, - foreign key (reference) references ValidPaths(id) on delete restrict -); - -create index if not exists IndexReferrer on Refs(referrer); -create index if not exists IndexReference on Refs(reference); - --- Paths can refer to themselves, causing a tuple (N, N) in the Refs --- table. This causes a deletion of the corresponding row in --- ValidPaths to cause a foreign key constraint violation (due to `on --- delete restrict' on the `reference' column). Therefore, explicitly --- get rid of self-references. -create trigger if not exists DeleteSelfRefs before delete on ValidPaths - begin - delete from Refs where referrer = old.id and reference = old.id; - end; - -create table if not exists DerivationOutputs ( - drv integer not null, - id text not null, -- symbolic output id, usually "out" - path text not null, - primary key (drv, id), - foreign key (drv) references ValidPaths(id) on delete cascade -); - -create index if not exists IndexDerivationOutputs on DerivationOutputs(path); - -create table if not exists FailedPaths ( - path text primary key not null, - time integer not null -); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc deleted file mode 100644 index 0238e5b0b6..0000000000 --- a/src/libstore/store-api.cc +++ /dev/null @@ -1,331 +0,0 @@ -#include "store-api.hh" -#include "globals.hh" -#include "util.hh" - -#include <climits> - - -namespace nix { - - -GCOptions::GCOptions() -{ - action = gcDeleteDead; - ignoreLiveness = false; - maxFreed = ULLONG_MAX; -} - - -bool isInStore(const Path & path) -{ - return isInDir(path, settings.nixStore); -} - - -bool isStorePath(const Path & path) -{ - return isInStore(path) - && path.find('/', settings.nixStore.size() + 1) == Path::npos; -} - - -void assertStorePath(const Path & path) -{ - if (!isStorePath(path)) - throw Error(format("path `%1%' is not in the Nix store") % path); -} - - -Path toStorePath(const Path & path) -{ - if (!isInStore(path)) - throw Error(format("path `%1%' is not in the Nix store") % path); - Path::size_type slash = path.find('/', settings.nixStore.size() + 1); - if (slash == Path::npos) - return path; - else - return Path(path, 0, slash); -} - - -Path followLinksToStore(const Path & _path) -{ - Path path = absPath(_path); - while (!isInStore(path)) { - if (!isLink(path)) break; - string target = readLink(path); - path = absPath(target, dirOf(path)); - } - if (!isInStore(path)) - throw Error(format("path `%1%' is not in the Nix store") % path); - return path; -} - - -Path followLinksToStorePath(const Path & path) -{ - return toStorePath(followLinksToStore(path)); -} - - -string storePathToName(const Path & path) -{ - assertStorePath(path); - return string(path, settings.nixStore.size() + 34); -} - - -void checkStoreName(const string & name) -{ - string validChars = "+-._?="; - /* Disallow names starting with a dot for possible security - reasons (e.g., "." and ".."). */ - if (string(name, 0, 1) == ".") - throw Error(format("illegal name: `%1%'") % name); - foreach (string::const_iterator, i, name) - if (!((*i >= 'A' && *i <= 'Z') || - (*i >= 'a' && *i <= 'z') || - (*i >= '0' && *i <= '9') || - validChars.find(*i) != string::npos)) - { - throw Error(format("invalid character `%1%' in name `%2%'") - % *i % name); - } -} - - -/* Store paths have the following form: - - <store>/<h>-<name> - - where - - <store> = the location of the Nix store, usually /nix/store - - <name> = a human readable name for the path, typically obtained - from the name attribute of the derivation, or the name of the - source file from which the store path is created. For derivation - outputs other than the default "out" output, the string "-<id>" - is suffixed to <name>. - - <h> = base-32 representation of the first 160 bits of a SHA-256 - hash of <s>; the hash part of the store name - - <s> = the string "<type>:sha256:<h2>:<store>:<name>"; - note that it includes the location of the store as well as the - name to make sure that changes to either of those are reflected - in the hash (e.g. you won't get /nix/store/<h>-name1 and - /nix/store/<h>-name2 with equal hash parts). - - <type> = one of: - "text:<r1>:<r2>:...<rN>" - for plain text files written to the store using - addTextToStore(); <r1> ... <rN> are the references of the - path. - "source" - for paths copied to the store using addToStore() when recursive - = true and hashAlgo = "sha256" - "output:<id>" - for either the outputs created by derivations, OR paths copied - to the store using addToStore() with recursive != true or - hashAlgo != "sha256" (in that case "source" is used; it's - silly, but it's done that way for compatibility). <id> is the - name of the output (usually, "out"). - - <h2> = base-16 representation of a SHA-256 hash of: - if <type> = "text:...": - the string written to the resulting store path - if <type> = "source": - the serialisation of the path from which this store path is - copied, as returned by hashPath() - if <type> = "output:out": - for non-fixed derivation outputs: - the derivation (see hashDerivationModulo() in - primops.cc) - for paths copied by addToStore() or produced by fixed-output - derivations: - the string "fixed:out:<rec><algo>:<hash>:", where - <rec> = "r:" for recursive (path) hashes, or "" or flat - (file) hashes - <algo> = "md5", "sha1" or "sha256" - <hash> = base-16 representation of the path or flat hash of - the contents of the path (or expected contents of the - path for fixed-output derivations) - - It would have been nicer to handle fixed-output derivations under - "source", e.g. have something like "source:<rec><algo>", but we're - stuck with this for now... - - The main reason for this way of computing names is to prevent name - collisions (for security). For instance, it shouldn't be feasible - to come up with a derivation whose output path collides with the - path for a copied source. The former would have a <s> starting with - "output:out:", while the latter would have a <2> starting with - "source:". -*/ - - -Path makeStorePath(const string & type, - const Hash & hash, const string & name) -{ - /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ - string s = type + ":sha256:" + printHash(hash) + ":" - + settings.nixStore + ":" + name; - - checkStoreName(name); - - return settings.nixStore + "/" - + printHash32(compressHash(hashString(htSHA256, s), 20)) - + "-" + name; -} - - -Path makeOutputPath(const string & id, - const Hash & hash, const string & name) -{ - return makeStorePath("output:" + id, hash, - name + (id == "out" ? "" : "-" + id)); -} - - -Path makeFixedOutputPath(bool recursive, - HashType hashAlgo, Hash hash, string name) -{ - return hashAlgo == htSHA256 && recursive - ? makeStorePath("source", hash, name) - : makeStorePath("output:out", hashString(htSHA256, - "fixed:out:" + (recursive ? (string) "r:" : "") + - printHashType(hashAlgo) + ":" + printHash(hash) + ":"), - name); -} - - -std::pair<Path, Hash> computeStorePathForPath(const Path & srcPath, - bool recursive, HashType hashAlgo, PathFilter & filter) -{ - HashType ht(hashAlgo); - Hash h = recursive ? hashPath(ht, srcPath, filter).first : hashFile(ht, srcPath); - string name = baseNameOf(srcPath); - Path dstPath = makeFixedOutputPath(recursive, hashAlgo, h, name); - return std::pair<Path, Hash>(dstPath, h); -} - - -Path computeStorePathForText(const string & name, const string & s, - const PathSet & references) -{ - Hash hash = hashString(htSHA256, s); - /* Stuff the references (if any) into the type. This is a bit - hacky, but we can't put them in `s' since that would be - ambiguous. */ - string type = "text"; - foreach (PathSet::const_iterator, i, references) { - type += ":"; - type += *i; - } - return makeStorePath(type, hash, name); -} - - -/* Return a string accepted by decodeValidPathInfo() that - registers the specified paths as valid. Note: it's the - responsibility of the caller to provide a closure. */ -string StoreAPI::makeValidityRegistration(const PathSet & paths, - bool showDerivers, bool showHash) -{ - string s = ""; - - foreach (PathSet::iterator, i, paths) { - s += *i + "\n"; - - ValidPathInfo info = queryPathInfo(*i); - - if (showHash) { - s += printHash(info.hash) + "\n"; - s += (format("%1%\n") % info.narSize).str(); - } - - Path deriver = showDerivers ? info.deriver : ""; - s += deriver + "\n"; - - s += (format("%1%\n") % info.references.size()).str(); - - foreach (PathSet::iterator, j, info.references) - s += *j + "\n"; - } - - return s; -} - - -ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven) -{ - ValidPathInfo info; - getline(str, info.path); - if (str.eof()) { info.path = ""; return info; } - if (hashGiven) { - string s; - getline(str, s); - info.hash = parseHash(htSHA256, s); - getline(str, s); - if (!string2Int(s, info.narSize)) throw Error("number expected"); - } - getline(str, info.deriver); - string s; int n; - getline(str, s); - if (!string2Int(s, n)) throw Error("number expected"); - while (n--) { - getline(str, s); - info.references.insert(s); - } - if (!str || str.eof()) throw Error("missing input"); - return info; -} - - -string showPaths(const PathSet & paths) -{ - string s; - foreach (PathSet::const_iterator, i, paths) { - if (s.size() != 0) s += ", "; - s += "`" + *i + "'"; - } - return s; -} - - -void exportPaths(StoreAPI & store, const Paths & paths, - bool sign, Sink & sink) -{ - foreach (Paths::const_iterator, i, paths) { - writeInt(1, sink); - store.exportPath(*i, sign, sink); - } - writeInt(0, sink); -} - - -} - - -#include "local-store.hh" -#include "serialise.hh" -#include "remote-store.hh" - - -namespace nix { - - -std::shared_ptr<StoreAPI> store; - - -std::shared_ptr<StoreAPI> openStore(bool reserveSpace) -{ - if (getEnv("NIX_REMOTE") == "") - return std::shared_ptr<StoreAPI>(new LocalStore(reserveSpace)); - else - return std::shared_ptr<StoreAPI>(new RemoteStore()); -} - - -} diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh deleted file mode 100644 index b635fee2cf..0000000000 --- a/src/libstore/store-api.hh +++ /dev/null @@ -1,366 +0,0 @@ -#pragma once - -#include "hash.hh" -#include "serialise.hh" - -#include <string> -#include <map> -#include <memory> - - -namespace nix { - - -typedef std::map<Path, Path> Roots; - - -struct GCOptions -{ - /* Garbage collector operation: - - - `gcReturnLive': return the set of paths reachable from - (i.e. in the closure of) the roots. - - - `gcReturnDead': return the set of paths not reachable from - the roots. - - - `gcDeleteDead': actually delete the latter set. - - - `gcDeleteSpecific': delete the paths listed in - `pathsToDelete', insofar as they are not reachable. - */ - typedef enum { - gcReturnLive, - gcReturnDead, - gcDeleteDead, - gcDeleteSpecific, - } GCAction; - - GCAction action; - - /* If `ignoreLiveness' is set, then reachability from the roots is - ignored (dangerous!). However, the paths must still be - unreferenced *within* the store (i.e., there can be no other - store paths that depend on them). */ - bool ignoreLiveness; - - /* For `gcDeleteSpecific', the paths to delete. */ - PathSet pathsToDelete; - - /* Stop after at least `maxFreed' bytes have been freed. */ - unsigned long long maxFreed; - - GCOptions(); -}; - - -struct GCResults -{ - /* Depending on the action, the GC roots, or the paths that would - be or have been deleted. */ - PathSet paths; - - /* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the - number of bytes that would be or was freed. */ - unsigned long long bytesFreed; - - GCResults() - { - bytesFreed = 0; - } -}; - - -struct SubstitutablePathInfo -{ - Path deriver; - PathSet references; - unsigned long long downloadSize; /* 0 = unknown or inapplicable */ - unsigned long long narSize; /* 0 = unknown */ -}; - -typedef std::map<Path, SubstitutablePathInfo> SubstitutablePathInfos; - - -struct ValidPathInfo -{ - Path path; - Path deriver; - Hash hash; - PathSet references; - time_t registrationTime; - unsigned long long narSize; // 0 = unknown - unsigned long long id; // internal use only - ValidPathInfo() : registrationTime(0), narSize(0) { } -}; - -typedef list<ValidPathInfo> ValidPathInfos; - - -enum BuildMode { bmNormal, bmRepair, bmCheck }; - - -class StoreAPI -{ -public: - - virtual ~StoreAPI() { } - - /* Check whether a path is valid. */ - virtual bool isValidPath(const Path & path) = 0; - - /* Query which of the given paths is valid. */ - virtual PathSet queryValidPaths(const PathSet & paths) = 0; - - /* Query the set of all valid paths. */ - virtual PathSet queryAllValidPaths() = 0; - - /* Query information about a valid path. */ - virtual ValidPathInfo queryPathInfo(const Path & path) = 0; - - /* Query the hash of a valid path. */ - virtual Hash queryPathHash(const Path & path) = 0; - - /* Query the set of outgoing FS references for a store path. The - result is not cleared. */ - virtual void queryReferences(const Path & path, - PathSet & references) = 0; - - /* Queries the set of incoming FS references for a store path. - The result is not cleared. */ - virtual void queryReferrers(const Path & path, - PathSet & referrers) = 0; - - /* Query the deriver of a store path. Return the empty string if - no deriver has been set. */ - virtual Path queryDeriver(const Path & path) = 0; - - /* Return all currently valid derivations that have `path' as an - output. (Note that the result of `queryDeriver()' is the - derivation that was actually used to produce `path', which may - not exist anymore.) */ - virtual PathSet queryValidDerivers(const Path & path) = 0; - - /* Query the outputs of the derivation denoted by `path'. */ - virtual PathSet queryDerivationOutputs(const Path & path) = 0; - - /* Query the output names of the derivation denoted by `path'. */ - virtual StringSet queryDerivationOutputNames(const Path & path) = 0; - - /* Query the full store path given the hash part of a valid store - path, or "" if the path doesn't exist. */ - virtual Path queryPathFromHashPart(const string & hashPart) = 0; - - /* Query which of the given paths have substitutes. */ - virtual PathSet querySubstitutablePaths(const PathSet & paths) = 0; - - /* Query substitute info (i.e. references, derivers and download - sizes) of a set of paths. If a path does not have substitute - info, it's omitted from the resulting âinfosâ map. */ - virtual void querySubstitutablePathInfos(const PathSet & paths, - SubstitutablePathInfos & infos) = 0; - - /* Copy the contents of a path to the store and register the - validity the resulting path. The resulting path is returned. - The function object `filter' can be used to exclude files (see - libutil/archive.hh). */ - virtual Path addToStore(const Path & srcPath, - bool recursive = true, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, bool repair = false) = 0; - - /* Like addToStore, but the contents written to the output path is - a regular file containing the given string. */ - virtual Path addTextToStore(const string & name, const string & s, - const PathSet & references, bool repair = false) = 0; - - /* Export a store path, that is, create a NAR dump of the store - path and append its references and its deriver. Optionally, a - cryptographic signature (created by OpenSSL) of the preceding - data is attached. */ - virtual void exportPath(const Path & path, bool sign, - Sink & sink) = 0; - - /* Import a sequence of NAR dumps created by exportPaths() into - the Nix store. */ - virtual Paths importPaths(bool requireSignature, Source & source) = 0; - - /* For each path, if it's a derivation, build it. Building a - derivation means ensuring that the output paths are valid. If - they are already valid, this is a no-op. Otherwise, validity - can be reached in two ways. First, if the output paths is - substitutable, then build the path that way. Second, the - output paths can be created by running the builder, after - recursively building any sub-derivations. For inputs that are - not derivations, substitute them. */ - virtual void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal) = 0; - - /* Ensure that a path is valid. If it is not currently valid, it - may be made valid by running a substitute (if defined for the - path). */ - virtual void ensurePath(const Path & path) = 0; - - /* Add a store path as a temporary root of the garbage collector. - The root disappears as soon as we exit. */ - virtual void addTempRoot(const Path & path) = 0; - - /* Add an indirect root, which is merely a symlink to `path' from - /nix/var/nix/gcroots/auto/<hash of `path'>. `path' is supposed - to be a symlink to a store path. The garbage collector will - automatically remove the indirect root when it finds that - `path' has disappeared. */ - virtual void addIndirectRoot(const Path & path) = 0; - - /* Acquire the global GC lock, then immediately release it. This - function must be called after registering a new permanent root, - but before exiting. Otherwise, it is possible that a running - garbage collector doesn't see the new root and deletes the - stuff we've just built. By acquiring the lock briefly, we - ensure that either: - - - The collector is already running, and so we block until the - collector is finished. The collector will know about our - *temporary* locks, which should include whatever it is we - want to register as a permanent lock. - - - The collector isn't running, or it's just started but hasn't - acquired the GC lock yet. In that case we get and release - the lock right away, then exit. The collector scans the - permanent root and sees our's. - - In either case the permanent root is seen by the collector. */ - virtual void syncWithGC() = 0; - - /* Find the roots of the garbage collector. Each root is a pair - (link, storepath) where `link' is the path of the symlink - outside of the Nix store that point to `storePath'. */ - virtual Roots findRoots() = 0; - - /* Perform a garbage collection. */ - virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0; - - /* Return the set of paths that have failed to build.*/ - virtual PathSet queryFailedPaths() = 0; - - /* Clear the "failed" status of the given paths. The special - value `*' causes all failed paths to be cleared. */ - virtual void clearFailedPaths(const PathSet & paths) = 0; - - /* Return a string representing information about the path that - can be loaded into the database using `nix-store --load-db' or - `nix-store --register-validity'. */ - string makeValidityRegistration(const PathSet & paths, - bool showDerivers, bool showHash); -}; - - -/* !!! These should be part of the store API, I guess. */ - -/* Throw an exception if `path' is not directly in the Nix store. */ -void assertStorePath(const Path & path); - -bool isInStore(const Path & path); -bool isStorePath(const Path & path); - -/* Extract the name part of the given store path. */ -string storePathToName(const Path & path); - -void checkStoreName(const string & name); - - -/* Chop off the parts after the top-level store name, e.g., - /nix/store/abcd-foo/bar => /nix/store/abcd-foo. */ -Path toStorePath(const Path & path); - - -/* Follow symlinks until we end up with a path in the Nix store. */ -Path followLinksToStore(const Path & path); - - -/* Same as followLinksToStore(), but apply toStorePath() to the - result. */ -Path followLinksToStorePath(const Path & path); - - -/* Constructs a unique store path name. */ -Path makeStorePath(const string & type, - const Hash & hash, const string & name); - -Path makeOutputPath(const string & id, - const Hash & hash, const string & name); - -Path makeFixedOutputPath(bool recursive, - HashType hashAlgo, Hash hash, string name); - - -/* This is the preparatory part of addToStore() and addToStoreFixed(); - it computes the store path to which srcPath is to be copied. - Returns the store path and the cryptographic hash of the - contents of srcPath. */ -std::pair<Path, Hash> computeStorePathForPath(const Path & srcPath, - bool recursive = true, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter); - -/* Preparatory part of addTextToStore(). - - !!! Computation of the path should take the references given to - addTextToStore() into account, otherwise we have a (relatively - minor) security hole: a caller can register a source file with - bogus references. If there are too many references, the path may - not be garbage collected when it has to be (not really a problem, - the caller could create a root anyway), or it may be garbage - collected when it shouldn't be (more serious). - - Hashing the references would solve this (bogus references would - simply yield a different store path, so other users wouldn't be - affected), but it has some backwards compatibility issues (the - hashing scheme changes), so I'm not doing that for now. */ -Path computeStorePathForText(const string & name, const string & s, - const PathSet & references); - - -/* Remove the temporary roots file for this process. Any temporary - root becomes garbage after this point unless it has been registered - as a (permanent) root. */ -void removeTempRoots(); - - -/* Register a permanent GC root. */ -Path addPermRoot(StoreAPI & store, const Path & storePath, - const Path & gcRoot, bool indirect, bool allowOutsideRootsDir = false); - - -/* Sort a set of paths topologically under the references relation. - If p refers to q, then p preceeds q in this list. */ -Paths topoSortPaths(StoreAPI & store, const PathSet & paths); - - -/* For now, there is a single global store API object, but we'll - purify that in the future. */ -extern std::shared_ptr<StoreAPI> store; - - -/* Factory method: open the Nix database, either through the local or - remote implementation. */ -std::shared_ptr<StoreAPI> openStore(bool reserveSpace = true); - - -/* Display a set of paths in human-readable form (i.e., between quotes - and separated by commas). */ -string showPaths(const PathSet & paths); - - -ValidPathInfo decodeValidPathInfo(std::istream & str, - bool hashGiven = false); - - -/* Export multiple paths in the format expected by ânix-store - --importâ. */ -void exportPaths(StoreAPI & store, const Paths & paths, - bool sign, Sink & sink); - - -MakeError(SubstError, Error) -MakeError(BuildError, Error) /* denotes a permanent build failure */ - - -} diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh deleted file mode 100644 index 9317f89c37..0000000000 --- a/src/libstore/worker-protocol.hh +++ /dev/null @@ -1,60 +0,0 @@ -#pragma once - -namespace nix { - - -#define WORKER_MAGIC_1 0x6e697863 -#define WORKER_MAGIC_2 0x6478696f - -#define PROTOCOL_VERSION 0x10e -#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) -#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) - - -typedef enum { - wopQuit = 0, - wopIsValidPath = 1, - wopHasSubstitutes = 3, - wopQueryPathHash = 4, - wopQueryReferences = 5, - wopQueryReferrers = 6, - wopAddToStore = 7, - wopAddTextToStore = 8, - wopBuildPaths = 9, - wopEnsurePath = 10, - wopAddTempRoot = 11, - wopAddIndirectRoot = 12, - wopSyncWithGC = 13, - wopFindRoots = 14, - wopExportPath = 16, - wopQueryDeriver = 18, - wopSetOptions = 19, - wopCollectGarbage = 20, - wopQuerySubstitutablePathInfo = 21, - wopQueryDerivationOutputs = 22, - wopQueryAllValidPaths = 23, - wopQueryFailedPaths = 24, - wopClearFailedPaths = 25, - wopQueryPathInfo = 26, - wopImportPaths = 27, - wopQueryDerivationOutputNames = 28, - wopQueryPathFromHashPart = 29, - wopQuerySubstitutablePathInfos = 30, - wopQueryValidPaths = 31, - wopQuerySubstitutablePaths = 32, - wopQueryValidDerivers = 33, -} WorkerOp; - - -#define STDERR_NEXT 0x6f6c6d67 -#define STDERR_READ 0x64617461 // data needed from source -#define STDERR_WRITE 0x64617416 // data for sink -#define STDERR_LAST 0x616c7473 -#define STDERR_ERROR 0x63787470 - - -Path readStorePath(Source & from); -template<class T> T readStorePaths(Source & from); - - -} diff --git a/src/libutil/affinity.cc b/src/libutil/affinity.cc deleted file mode 100644 index 3e21f43a2e..0000000000 --- a/src/libutil/affinity.cc +++ /dev/null @@ -1,55 +0,0 @@ -#include "types.hh" -#include "util.hh" -#include "affinity.hh" - -#if HAVE_SCHED_H -#include <sched.h> -#endif - -namespace nix { - - -#if HAVE_SCHED_SETAFFINITY -static bool didSaveAffinity = false; -static cpu_set_t savedAffinity; -#endif - - -void setAffinityTo(int cpu) -{ -#if HAVE_SCHED_SETAFFINITY - if (sched_getaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) return; - didSaveAffinity = true; - printMsg(lvlDebug, format("locking this thread to CPU %1%") % cpu); - cpu_set_t newAffinity; - CPU_ZERO(&newAffinity); - CPU_SET(cpu, &newAffinity); - if (sched_setaffinity(0, sizeof(cpu_set_t), &newAffinity) == -1) - printMsg(lvlError, format("failed to lock thread to CPU %1%") % cpu); -#endif -} - - -int lockToCurrentCPU() -{ -#if HAVE_SCHED_SETAFFINITY - int cpu = sched_getcpu(); - if (cpu != -1) setAffinityTo(cpu); - return cpu; -#else - return -1; -#endif -} - - -void restoreAffinity() -{ -#if HAVE_SCHED_SETAFFINITY - if (!didSaveAffinity) return; - if (sched_setaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) - printMsg(lvlError, "failed to restore affinity %1%"); -#endif -} - - -} diff --git a/src/libutil/affinity.hh b/src/libutil/affinity.hh deleted file mode 100644 index c1bd28e136..0000000000 --- a/src/libutil/affinity.hh +++ /dev/null @@ -1,9 +0,0 @@ -#pragma once - -namespace nix { - -void setAffinityTo(int cpu); -int lockToCurrentCPU(); -void restoreAffinity(); - -} diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc deleted file mode 100644 index ab4cd47351..0000000000 --- a/src/libutil/archive.cc +++ /dev/null @@ -1,335 +0,0 @@ -#include "config.h" - -#include <cerrno> -#include <algorithm> -#include <vector> - -#define _XOPEN_SOURCE 600 -#include <sys/types.h> -#include <sys/stat.h> -#include <unistd.h> -#include <dirent.h> -#include <fcntl.h> - -#include "archive.hh" -#include "util.hh" - - -namespace nix { - - -static string archiveVersion1 = "nix-archive-1"; - - -PathFilter defaultPathFilter; - - -static void dump(const string & path, Sink & sink, PathFilter & filter); - - -static void dumpEntries(const Path & path, Sink & sink, PathFilter & filter) -{ - Strings names = readDirectory(path); - vector<string> names2(names.begin(), names.end()); - sort(names2.begin(), names2.end()); - - for (vector<string>::iterator i = names2.begin(); - i != names2.end(); ++i) - { - Path entry = path + "/" + *i; - if (filter(entry)) { - writeString("entry", sink); - writeString("(", sink); - writeString("name", sink); - writeString(*i, sink); - writeString("node", sink); - dump(entry, sink, filter); - writeString(")", sink); - } - } -} - - -static void dumpContents(const Path & path, size_t size, - Sink & sink) -{ - writeString("contents", sink); - writeLongLong(size, sink); - - AutoCloseFD fd = open(path.c_str(), O_RDONLY); - if (fd == -1) throw SysError(format("opening file `%1%'") % path); - - unsigned char buf[65536]; - size_t left = size; - - while (left > 0) { - size_t n = left > sizeof(buf) ? sizeof(buf) : left; - readFull(fd, buf, n); - left -= n; - sink(buf, n); - } - - writePadding(size, sink); -} - - -static void dump(const Path & path, Sink & sink, PathFilter & filter) -{ - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path `%1%'") % path); - - writeString("(", sink); - - if (S_ISREG(st.st_mode)) { - writeString("type", sink); - writeString("regular", sink); - if (st.st_mode & S_IXUSR) { - writeString("executable", sink); - writeString("", sink); - } - dumpContents(path, (size_t) st.st_size, sink); - } - - else if (S_ISDIR(st.st_mode)) { - writeString("type", sink); - writeString("directory", sink); - dumpEntries(path, sink, filter); - } - - else if (S_ISLNK(st.st_mode)) { - writeString("type", sink); - writeString("symlink", sink); - writeString("target", sink); - writeString(readLink(path), sink); - } - - else throw Error(format("file `%1%' has an unknown type") % path); - - writeString(")", sink); -} - - -void dumpPath(const Path & path, Sink & sink, PathFilter & filter) -{ - writeString(archiveVersion1, sink); - dump(path, sink, filter); -} - - -static SerialisationError badArchive(string s) -{ - return SerialisationError("bad archive: " + s); -} - - -static void skipGeneric(Source & source) -{ - if (readString(source) == "(") { - while (readString(source) != ")") - skipGeneric(source); - } -} - - -static void parse(ParseSink & sink, Source & source, const Path & path); - - - -static void parseEntry(ParseSink & sink, Source & source, const Path & path) -{ - string s, name; - - s = readString(source); - if (s != "(") throw badArchive("expected open tag"); - - while (1) { - checkInterrupt(); - - s = readString(source); - - if (s == ")") { - break; - } else if (s == "name") { - name = readString(source); - } else if (s == "node") { - if (s == "") throw badArchive("entry name missing"); - parse(sink, source, path + "/" + name); - } else { - throw badArchive("unknown field " + s); - skipGeneric(source); - } - } -} - - -static void parseContents(ParseSink & sink, Source & source, const Path & path) -{ - unsigned long long size = readLongLong(source); - - sink.preallocateContents(size); - - unsigned long long left = size; - unsigned char buf[65536]; - - while (left) { - checkInterrupt(); - unsigned int n = sizeof(buf); - if ((unsigned long long) n > left) n = left; - source(buf, n); - sink.receiveContents(buf, n); - left -= n; - } - - readPadding(size, source); -} - - -static void parse(ParseSink & sink, Source & source, const Path & path) -{ - string s; - - s = readString(source); - if (s != "(") throw badArchive("expected open tag"); - - enum { tpUnknown, tpRegular, tpDirectory, tpSymlink } type = tpUnknown; - - while (1) { - checkInterrupt(); - - s = readString(source); - - if (s == ")") { - break; - } - - else if (s == "type") { - if (type != tpUnknown) - throw badArchive("multiple type fields"); - string t = readString(source); - - if (t == "regular") { - type = tpRegular; - sink.createRegularFile(path); - } - - else if (t == "directory") { - sink.createDirectory(path); - type = tpDirectory; - } - - else if (t == "symlink") { - type = tpSymlink; - } - - else throw badArchive("unknown file type " + t); - - } - - else if (s == "contents" && type == tpRegular) { - parseContents(sink, source, path); - } - - else if (s == "executable" && type == tpRegular) { - readString(source); - sink.isExecutable(); - } - - else if (s == "entry" && type == tpDirectory) { - parseEntry(sink, source, path); - } - - else if (s == "target" && type == tpSymlink) { - string target = readString(source); - sink.createSymlink(path, target); - } - - else { - throw badArchive("unknown field " + s); - skipGeneric(source); - } - } -} - - -void parseDump(ParseSink & sink, Source & source) -{ - string version; - try { - version = readString(source); - } catch (SerialisationError & e) { - /* This generally means the integer at the start couldn't be - decoded. Ignore and throw the exception below. */ - } - if (version != archiveVersion1) - throw badArchive("input doesn't look like a Nix archive"); - parse(sink, source, ""); -} - - -struct RestoreSink : ParseSink -{ - Path dstPath; - AutoCloseFD fd; - - void createDirectory(const Path & path) - { - Path p = dstPath + path; - if (mkdir(p.c_str(), 0777) == -1) - throw SysError(format("creating directory `%1%'") % p); - }; - - void createRegularFile(const Path & path) - { - Path p = dstPath + path; - fd.close(); - fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY, 0666); - if (fd == -1) throw SysError(format("creating file `%1%'") % p); - } - - void isExecutable() - { - struct stat st; - if (fstat(fd, &st) == -1) - throw SysError("fstat"); - if (fchmod(fd, st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1) - throw SysError("fchmod"); - } - - void preallocateContents(unsigned long long len) - { -#if HAVE_POSIX_FALLOCATE - if (len) { - errno = posix_fallocate(fd, 0, len); - /* Note that EINVAL may indicate that the underlying - filesystem doesn't support preallocation (e.g. on - OpenSolaris). Since preallocation is just an - optimisation, ignore it. */ - if (errno && errno != EINVAL) - throw SysError(format("preallocating file of %1% bytes") % len); - } -#endif - } - - void receiveContents(unsigned char * data, unsigned int len) - { - writeFull(fd, data, len); - } - - void createSymlink(const Path & path, const string & target) - { - Path p = dstPath + path; - nix::createSymlink(target, p); - } -}; - - -void restorePath(const Path & path, Source & source) -{ - RestoreSink sink; - sink.dstPath = path; - parseDump(sink, source); -} - - -} diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh deleted file mode 100644 index ccac92074d..0000000000 --- a/src/libutil/archive.hh +++ /dev/null @@ -1,75 +0,0 @@ -#pragma once - -#include "types.hh" -#include "serialise.hh" - - -namespace nix { - - -/* dumpPath creates a Nix archive of the specified path. The format - is as follows: - - IF path points to a REGULAR FILE: - dump(path) = attrs( - [ ("type", "regular") - , ("contents", contents(path)) - ]) - - IF path points to a DIRECTORY: - dump(path) = attrs( - [ ("type", "directory") - , ("entries", concat(map(f, sort(entries(path))))) - ]) - where f(fn) = attrs( - [ ("name", fn) - , ("file", dump(path + "/" + fn)) - ]) - - where: - - attrs(as) = concat(map(attr, as)) + encN(0) - attrs((a, b)) = encS(a) + encS(b) - - encS(s) = encN(len(s)) + s + (padding until next 64-bit boundary) - - encN(n) = 64-bit little-endian encoding of n. - - contents(path) = the contents of a regular file. - - sort(strings) = lexicographic sort by 8-bit value (strcmp). - - entries(path) = the entries of a directory, without `.' and - `..'. - - `+' denotes string concatenation. */ - -struct PathFilter -{ - virtual ~PathFilter() { } - virtual bool operator () (const Path & path) { return true; } -}; - -extern PathFilter defaultPathFilter; - -void dumpPath(const Path & path, Sink & sink, - PathFilter & filter = defaultPathFilter); - -struct ParseSink -{ - virtual void createDirectory(const Path & path) { }; - - virtual void createRegularFile(const Path & path) { }; - virtual void isExecutable() { }; - virtual void preallocateContents(unsigned long long size) { }; - virtual void receiveContents(unsigned char * data, unsigned int len) { }; - - virtual void createSymlink(const Path & path, const string & target) { }; -}; - -void parseDump(ParseSink & sink, Source & source); - -void restorePath(const Path & path, Source & source); - - -} diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc deleted file mode 100644 index de2c1ebd72..0000000000 --- a/src/libutil/hash.cc +++ /dev/null @@ -1,382 +0,0 @@ -#include "config.h" - -#include <iostream> -#include <cstring> - -#ifdef HAVE_OPENSSL -#include <openssl/md5.h> -#include <openssl/sha.h> -#else -extern "C" { -#include "md5.h" -#include "sha1.h" -#include "sha256.h" -} -#endif - -#include "hash.hh" -#include "archive.hh" -#include "util.hh" - -#include <sys/types.h> -#include <sys/stat.h> -#include <fcntl.h> - - -namespace nix { - - -Hash::Hash() -{ - type = htUnknown; - hashSize = 0; - memset(hash, 0, maxHashSize); -} - - -Hash::Hash(HashType type) -{ - this->type = type; - if (type == htMD5) hashSize = md5HashSize; - else if (type == htSHA1) hashSize = sha1HashSize; - else if (type == htSHA256) hashSize = sha256HashSize; - else throw Error("unknown hash type"); - assert(hashSize <= maxHashSize); - memset(hash, 0, maxHashSize); -} - - -bool Hash::operator == (const Hash & h2) const -{ - if (hashSize != h2.hashSize) return false; - for (unsigned int i = 0; i < hashSize; i++) - if (hash[i] != h2.hash[i]) return false; - return true; -} - - -bool Hash::operator != (const Hash & h2) const -{ - return !(*this == h2); -} - - -bool Hash::operator < (const Hash & h) const -{ - for (unsigned int i = 0; i < hashSize; i++) { - if (hash[i] < h.hash[i]) return true; - if (hash[i] > h.hash[i]) return false; - } - return false; -} - - -const string base16Chars = "0123456789abcdef"; - - -string printHash(const Hash & hash) -{ - char buf[hash.hashSize * 2]; - for (unsigned int i = 0; i < hash.hashSize; i++) { - buf[i * 2] = base16Chars[hash.hash[i] >> 4]; - buf[i * 2 + 1] = base16Chars[hash.hash[i] & 0x0f]; - } - return string(buf, hash.hashSize * 2); -} - - -Hash parseHash(HashType ht, const string & s) -{ - Hash hash(ht); - if (s.length() != hash.hashSize * 2) - throw Error(format("invalid hash `%1%'") % s); - for (unsigned int i = 0; i < hash.hashSize; i++) { - string s2(s, i * 2, 2); - if (!isxdigit(s2[0]) || !isxdigit(s2[1])) - throw Error(format("invalid hash `%1%'") % s); - std::istringstream str(s2); - int n; - str >> std::hex >> n; - hash.hash[i] = n; - } - return hash; -} - - -static unsigned char divMod(unsigned char * bytes, unsigned char y) -{ - unsigned int borrow = 0; - - int pos = Hash::maxHashSize - 1; - while (pos >= 0 && !bytes[pos]) --pos; - - for ( ; pos >= 0; --pos) { - unsigned int s = bytes[pos] + (borrow << 8); - unsigned int d = s / y; - borrow = s % y; - bytes[pos] = d; - } - - return borrow; -} - - -unsigned int hashLength32(const Hash & hash) -{ - return (hash.hashSize * 8 - 1) / 5 + 1; -} - - -// omitted: E O U T -const string base32Chars = "0123456789abcdfghijklmnpqrsvwxyz"; - - -string printHash32(const Hash & hash) -{ - Hash hash2(hash); - unsigned int len = hashLength32(hash); - - const char * chars = base32Chars.data(); - - string s(len, '0'); - - int pos = len - 1; - while (pos >= 0) { - unsigned char digit = divMod(hash2.hash, 32); - s[pos--] = chars[digit]; - } - - for (unsigned int i = 0; i < hash2.maxHashSize; ++i) - assert(hash2.hash[i] == 0); - - return s; -} - - -string printHash16or32(const Hash & hash) -{ - return hash.type == htMD5 ? printHash(hash) : printHash32(hash); -} - - -static bool mul(unsigned char * bytes, unsigned char y, int maxSize) -{ - unsigned char carry = 0; - - for (int pos = 0; pos < maxSize; ++pos) { - unsigned int m = bytes[pos] * y + carry; - bytes[pos] = m & 0xff; - carry = m >> 8; - } - - return carry; -} - - -static bool add(unsigned char * bytes, unsigned char y, int maxSize) -{ - unsigned char carry = y; - - for (int pos = 0; pos < maxSize; ++pos) { - unsigned int m = bytes[pos] + carry; - bytes[pos] = m & 0xff; - carry = m >> 8; - if (carry == 0) break; - } - - return carry; -} - - -Hash parseHash32(HashType ht, const string & s) -{ - Hash hash(ht); - - const char * chars = base32Chars.data(); - - for (unsigned int i = 0; i < s.length(); ++i) { - char c = s[i]; - unsigned char digit; - for (digit = 0; digit < base32Chars.size(); ++digit) /* !!! slow */ - if (chars[digit] == c) break; - if (digit >= 32) - throw Error(format("invalid base-32 hash `%1%'") % s); - if (mul(hash.hash, 32, hash.hashSize) || - add(hash.hash, digit, hash.hashSize)) - throw Error(format("base-32 hash `%1%' is too large") % s); - } - - return hash; -} - - -Hash parseHash16or32(HashType ht, const string & s) -{ - Hash hash(ht); - if (s.size() == hash.hashSize * 2) - /* hexadecimal representation */ - hash = parseHash(ht, s); - else if (s.size() == hashLength32(hash)) - /* base-32 representation */ - hash = parseHash32(ht, s); - else - throw Error(format("hash `%1%' has wrong length for hash type `%2%'") - % s % printHashType(ht)); - return hash; -} - - -bool isHash(const string & s) -{ - if (s.length() != 32) return false; - for (int i = 0; i < 32; i++) { - char c = s[i]; - if (!((c >= '0' && c <= '9') || - (c >= 'a' && c <= 'f'))) - return false; - } - return true; -} - - -union Ctx -{ - MD5_CTX md5; - SHA_CTX sha1; - SHA256_CTX sha256; -}; - - -static void start(HashType ht, Ctx & ctx) -{ - if (ht == htMD5) MD5_Init(&ctx.md5); - else if (ht == htSHA1) SHA1_Init(&ctx.sha1); - else if (ht == htSHA256) SHA256_Init(&ctx.sha256); -} - - -static void update(HashType ht, Ctx & ctx, - const unsigned char * bytes, unsigned int len) -{ - if (ht == htMD5) MD5_Update(&ctx.md5, bytes, len); - else if (ht == htSHA1) SHA1_Update(&ctx.sha1, bytes, len); - else if (ht == htSHA256) SHA256_Update(&ctx.sha256, bytes, len); -} - - -static void finish(HashType ht, Ctx & ctx, unsigned char * hash) -{ - if (ht == htMD5) MD5_Final(hash, &ctx.md5); - else if (ht == htSHA1) SHA1_Final(hash, &ctx.sha1); - else if (ht == htSHA256) SHA256_Final(hash, &ctx.sha256); -} - - -Hash hashString(HashType ht, const string & s) -{ - Ctx ctx; - Hash hash(ht); - start(ht, ctx); - update(ht, ctx, (const unsigned char *) s.data(), s.length()); - finish(ht, ctx, hash.hash); - return hash; -} - - -Hash hashFile(HashType ht, const Path & path) -{ - Ctx ctx; - Hash hash(ht); - start(ht, ctx); - - AutoCloseFD fd = open(path.c_str(), O_RDONLY); - if (fd == -1) throw SysError(format("opening file `%1%'") % path); - - unsigned char buf[8192]; - ssize_t n; - while ((n = read(fd, buf, sizeof(buf)))) { - checkInterrupt(); - if (n == -1) throw SysError(format("reading file `%1%'") % path); - update(ht, ctx, buf, n); - } - - finish(ht, ctx, hash.hash); - return hash; -} - - -HashSink::HashSink(HashType ht) : ht(ht) -{ - ctx = new Ctx; - bytes = 0; - start(ht, *ctx); -} - -HashSink::~HashSink() -{ - bufPos = 0; - delete ctx; -} - -void HashSink::write(const unsigned char * data, size_t len) -{ - bytes += len; - update(ht, *ctx, data, len); -} - -HashResult HashSink::finish() -{ - flush(); - Hash hash(ht); - nix::finish(ht, *ctx, hash.hash); - return HashResult(hash, bytes); -} - -HashResult HashSink::currentHash() -{ - flush(); - Ctx ctx2 = *ctx; - Hash hash(ht); - nix::finish(ht, ctx2, hash.hash); - return HashResult(hash, bytes); -} - - -HashResult hashPath( - HashType ht, const Path & path, PathFilter & filter) -{ - HashSink sink(ht); - dumpPath(path, sink, filter); - return sink.finish(); -} - - -Hash compressHash(const Hash & hash, unsigned int newSize) -{ - Hash h; - h.hashSize = newSize; - for (unsigned int i = 0; i < hash.hashSize; ++i) - h.hash[i % newSize] ^= hash.hash[i]; - return h; -} - - -HashType parseHashType(const string & s) -{ - if (s == "md5") return htMD5; - else if (s == "sha1") return htSHA1; - else if (s == "sha256") return htSHA256; - else return htUnknown; -} - - -string printHashType(HashType ht) -{ - if (ht == htMD5) return "md5"; - else if (ht == htSHA1) return "sha1"; - else if (ht == htSHA256) return "sha256"; - else throw Error("cannot print unknown hash type"); -} - - -} diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh deleted file mode 100644 index 2c6f176ec7..0000000000 --- a/src/libutil/hash.hh +++ /dev/null @@ -1,113 +0,0 @@ -#pragma once - -#include "types.hh" -#include "serialise.hh" - - -namespace nix { - - -typedef enum { htUnknown, htMD5, htSHA1, htSHA256 } HashType; - - -const int md5HashSize = 16; -const int sha1HashSize = 20; -const int sha256HashSize = 32; - -extern const string base32Chars; - - -struct Hash -{ - static const unsigned int maxHashSize = 32; - unsigned int hashSize; - unsigned char hash[maxHashSize]; - - HashType type; - - /* Create an unusable hash object. */ - Hash(); - - /* Create a zero-filled hash object. */ - Hash(HashType type); - - /* Check whether two hash are equal. */ - bool operator == (const Hash & h2) const; - - /* Check whether two hash are not equal. */ - bool operator != (const Hash & h2) const; - - /* For sorting. */ - bool operator < (const Hash & h) const; -}; - - -/* Convert a hash to a hexadecimal representation. */ -string printHash(const Hash & hash); - -/* Parse a hexadecimal representation of a hash code. */ -Hash parseHash(HashType ht, const string & s); - -/* Returns the length of a base-32 hash representation. */ -unsigned int hashLength32(const Hash & hash); - -/* Convert a hash to a base-32 representation. */ -string printHash32(const Hash & hash); - -/* Print a hash in base-16 if it's MD5, or base-32 otherwise. */ -string printHash16or32(const Hash & hash); - -/* Parse a base-32 representation of a hash code. */ -Hash parseHash32(HashType ht, const string & s); - -/* Parse a base-16 or base-32 representation of a hash code. */ -Hash parseHash16or32(HashType ht, const string & s); - -/* Verify that the given string is a valid hash code. */ -bool isHash(const string & s); - -/* Compute the hash of the given string. */ -Hash hashString(HashType ht, const string & s); - -/* Compute the hash of the given file. */ -Hash hashFile(HashType ht, const Path & path); - -/* Compute the hash of the given path. The hash is defined as - (essentially) hashString(ht, dumpPath(path)). */ -struct PathFilter; -extern PathFilter defaultPathFilter; -typedef std::pair<Hash, unsigned long long> HashResult; -HashResult hashPath(HashType ht, const Path & path, - PathFilter & filter = defaultPathFilter); - -/* Compress a hash to the specified number of bytes by cyclically - XORing bytes together. */ -Hash compressHash(const Hash & hash, unsigned int newSize); - -/* Parse a string representing a hash type. */ -HashType parseHashType(const string & s); - -/* And the reverse. */ -string printHashType(HashType ht); - - -union Ctx; - -class HashSink : public BufferedSink -{ -private: - HashType ht; - Ctx * ctx; - unsigned long long bytes; - -public: - HashSink(HashType ht); - HashSink(const HashSink & h); - ~HashSink(); - void write(const unsigned char * data, size_t len); - HashResult finish(); - HashResult currentHash(); -}; - - -} diff --git a/src/libutil/local.mk b/src/libutil/local.mk deleted file mode 100644 index 8af2e78d9c..0000000000 --- a/src/libutil/local.mk +++ /dev/null @@ -1,15 +0,0 @@ -libraries += libutil - -libutil_NAME = libnixutil - -libutil_DIR := $(d) - -libutil_SOURCES := $(wildcard $(d)/*.cc) - -ifeq ($(HAVE_OPENSSL), 1) - libutil_LDFLAGS = $(OPENSSL_LIBS) -else - libutil_SOURCES += $(d)/md5.c $(d)/sha1.c $(d)/sha256.c -endif - -libutil_LIBS = libformat diff --git a/src/libutil/md32_common.h b/src/libutil/md32_common.h deleted file mode 100644 index 0cbcfaf8a2..0000000000 --- a/src/libutil/md32_common.h +++ /dev/null @@ -1,620 +0,0 @@ -/* crypto/md32_common.h */ -/* ==================================================================== - * Copyright (c) 1999-2002 The OpenSSL Project. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * 3. All advertising materials mentioning features or use of this - * software must display the following acknowledgment: - * "This product includes software developed by the OpenSSL Project - * for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)" - * - * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to - * endorse or promote products derived from this software without - * prior written permission. For written permission, please contact - * licensing@OpenSSL.org. - * - * 5. Products derived from this software may not be called "OpenSSL" - * nor may "OpenSSL" appear in their names without prior written - * permission of the OpenSSL Project. - * - * 6. Redistributions of any form whatsoever must retain the following - * acknowledgment: - * "This product includes software developed by the OpenSSL Project - * for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)" - * - * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY - * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR - * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * ==================================================================== - * - * This product includes cryptographic software written by Eric Young - * (eay@cryptsoft.com). This product includes software written by Tim - * Hudson (tjh@cryptsoft.com). - * - */ - -/* - * This is a generic 32 bit "collector" for message digest algorithms. - * Whenever needed it collects input character stream into chunks of - * 32 bit values and invokes a block function that performs actual hash - * calculations. - * - * Porting guide. - * - * Obligatory macros: - * - * DATA_ORDER_IS_BIG_ENDIAN or DATA_ORDER_IS_LITTLE_ENDIAN - * this macro defines byte order of input stream. - * HASH_CBLOCK - * size of a unit chunk HASH_BLOCK operates on. - * HASH_LONG - * has to be at lest 32 bit wide, if it's wider, then - * HASH_LONG_LOG2 *has to* be defined along - * HASH_CTX - * context structure that at least contains following - * members: - * typedef struct { - * ... - * HASH_LONG Nl,Nh; - * HASH_LONG data[HASH_LBLOCK]; - * unsigned int num; - * ... - * } HASH_CTX; - * HASH_UPDATE - * name of "Update" function, implemented here. - * HASH_TRANSFORM - * name of "Transform" function, implemented here. - * HASH_FINAL - * name of "Final" function, implemented here. - * HASH_BLOCK_HOST_ORDER - * name of "block" function treating *aligned* input message - * in host byte order, implemented externally. - * HASH_BLOCK_DATA_ORDER - * name of "block" function treating *unaligned* input message - * in original (data) byte order, implemented externally (it - * actually is optional if data and host are of the same - * "endianess"). - * HASH_MAKE_STRING - * macro convering context variables to an ASCII hash string. - * - * Optional macros: - * - * B_ENDIAN or L_ENDIAN - * defines host byte-order. - * HASH_LONG_LOG2 - * defaults to 2 if not states otherwise. - * HASH_LBLOCK - * assumed to be HASH_CBLOCK/4 if not stated otherwise. - * HASH_BLOCK_DATA_ORDER_ALIGNED - * alternative "block" function capable of treating - * aligned input message in original (data) order, - * implemented externally. - * - * MD5 example: - * - * #define DATA_ORDER_IS_LITTLE_ENDIAN - * - * #define HASH_LONG MD5_LONG - * #define HASH_LONG_LOG2 MD5_LONG_LOG2 - * #define HASH_CTX MD5_CTX - * #define HASH_CBLOCK MD5_CBLOCK - * #define HASH_LBLOCK MD5_LBLOCK - * #define HASH_UPDATE MD5_Update - * #define HASH_TRANSFORM MD5_Transform - * #define HASH_FINAL MD5_Final - * #define HASH_BLOCK_HOST_ORDER md5_block_host_order - * #define HASH_BLOCK_DATA_ORDER md5_block_data_order - * - * <appro@fy.chalmers.se> - */ - -#if !defined(DATA_ORDER_IS_BIG_ENDIAN) && !defined(DATA_ORDER_IS_LITTLE_ENDIAN) -#error "DATA_ORDER must be defined!" -#endif - -#ifndef HASH_CBLOCK -#error "HASH_CBLOCK must be defined!" -#endif -#ifndef HASH_LONG -#error "HASH_LONG must be defined!" -#endif -#ifndef HASH_CTX -#error "HASH_CTX must be defined!" -#endif - -#ifndef HASH_UPDATE -#error "HASH_UPDATE must be defined!" -#endif -#ifndef HASH_TRANSFORM -#error "HASH_TRANSFORM must be defined!" -#endif -#ifndef HASH_FINAL -#error "HASH_FINAL must be defined!" -#endif - -#ifndef HASH_BLOCK_HOST_ORDER -#error "HASH_BLOCK_HOST_ORDER must be defined!" -#endif - -#if 0 -/* - * Moved below as it's required only if HASH_BLOCK_DATA_ORDER_ALIGNED - * isn't defined. - */ -#ifndef HASH_BLOCK_DATA_ORDER -#error "HASH_BLOCK_DATA_ORDER must be defined!" -#endif -#endif - -#ifndef HASH_LBLOCK -#define HASH_LBLOCK (HASH_CBLOCK/4) -#endif - -#ifndef HASH_LONG_LOG2 -#define HASH_LONG_LOG2 2 -#endif - -/* - * Engage compiler specific rotate intrinsic function if available. - */ -#undef ROTATE -#ifndef PEDANTIC -# if defined(_MSC_VER) || defined(__ICC) -# define ROTATE(a,n) _lrotl(a,n) -# elif defined(__MWERKS__) -# if defined(__POWERPC__) -# define ROTATE(a,n) __rlwinm(a,n,0,31) -# elif defined(__MC68K__) - /* Motorola specific tweak. <appro@fy.chalmers.se> */ -# define ROTATE(a,n) ( n<24 ? __rol(a,n) : __ror(a,32-n) ) -# else -# define ROTATE(a,n) __rol(a,n) -# endif -# elif defined(__GNUC__) && __GNUC__>=2 && !defined(OPENSSL_NO_ASM) && !defined(OPENSSL_NO_INLINE_ASM) - /* - * Some GNU C inline assembler templates. Note that these are - * rotates by *constant* number of bits! But that's exactly - * what we need here... - * <appro@fy.chalmers.se> - */ -# if defined(__i386) || defined(__i386__) || defined(__x86_64) || defined(__x86_64__) -# define ROTATE(a,n) ({ register unsigned int ret; \ - asm ( \ - "roll %1,%0" \ - : "=r"(ret) \ - : "I"(n), "0"(a) \ - : "cc"); \ - ret; \ - }) -# elif defined(__powerpc) || defined(__ppc__) || defined(__powerpc64__) -# define ROTATE(a,n) ({ register unsigned int ret; \ - asm ( \ - "rlwinm %0,%1,%2,0,31" \ - : "=r"(ret) \ - : "r"(a), "I"(n)); \ - ret; \ - }) -# endif -# endif -#endif /* PEDANTIC */ - -#if HASH_LONG_LOG2==2 /* Engage only if sizeof(HASH_LONG)== 4 */ -/* A nice byte order reversal from Wei Dai <weidai@eskimo.com> */ -#ifdef ROTATE -/* 5 instructions with rotate instruction, else 9 */ -#define REVERSE_FETCH32(a,l) ( \ - l=*(const HASH_LONG *)(a), \ - ((ROTATE(l,8)&0x00FF00FF)|(ROTATE((l&0x00FF00FF),24))) \ - ) -#else -/* 6 instructions with rotate instruction, else 8 */ -#define REVERSE_FETCH32(a,l) ( \ - l=*(const HASH_LONG *)(a), \ - l=(((l>>8)&0x00FF00FF)|((l&0x00FF00FF)<<8)), \ - ROTATE(l,16) \ - ) -/* - * Originally the middle line started with l=(((l&0xFF00FF00)>>8)|... - * It's rewritten as above for two reasons: - * - RISCs aren't good at long constants and have to explicitely - * compose 'em with several (well, usually 2) instructions in a - * register before performing the actual operation and (as you - * already realized:-) having same constant should inspire the - * compiler to permanently allocate the only register for it; - * - most modern CPUs have two ALUs, but usually only one has - * circuitry for shifts:-( this minor tweak inspires compiler - * to schedule shift instructions in a better way... - * - * <appro@fy.chalmers.se> - */ -#endif -#endif - -#ifndef ROTATE -#define ROTATE(a,n) (((a)<<(n))|(((a)&0xffffffff)>>(32-(n)))) -#endif - -/* - * Make some obvious choices. E.g., HASH_BLOCK_DATA_ORDER_ALIGNED - * and HASH_BLOCK_HOST_ORDER ought to be the same if input data - * and host are of the same "endianess". It's possible to mask - * this with blank #define HASH_BLOCK_DATA_ORDER though... - * - * <appro@fy.chalmers.se> - */ -#if defined(B_ENDIAN) -# if defined(DATA_ORDER_IS_BIG_ENDIAN) -# if !defined(HASH_BLOCK_DATA_ORDER_ALIGNED) && HASH_LONG_LOG2==2 -# define HASH_BLOCK_DATA_ORDER_ALIGNED HASH_BLOCK_HOST_ORDER -# endif -# endif -#elif defined(L_ENDIAN) -# if defined(DATA_ORDER_IS_LITTLE_ENDIAN) -# if !defined(HASH_BLOCK_DATA_ORDER_ALIGNED) && HASH_LONG_LOG2==2 -# define HASH_BLOCK_DATA_ORDER_ALIGNED HASH_BLOCK_HOST_ORDER -# endif -# endif -#endif - -#if !defined(HASH_BLOCK_DATA_ORDER_ALIGNED) -#ifndef HASH_BLOCK_DATA_ORDER -#error "HASH_BLOCK_DATA_ORDER must be defined!" -#endif -#endif - -#if defined(DATA_ORDER_IS_BIG_ENDIAN) - -#ifndef PEDANTIC -# if defined(__GNUC__) && __GNUC__>=2 && !defined(OPENSSL_NO_ASM) && !defined(OPENSSL_NO_INLINE_ASM) -# if defined(__i386) || defined(__i386__) || defined(__x86_64) || defined(__x86_64__) - /* - * This gives ~30-40% performance improvement in SHA-256 compiled - * with gcc [on P4]. Well, first macro to be frank. We can pull - * this trick on x86* platforms only, because these CPUs can fetch - * unaligned data without raising an exception. - */ -# define HOST_c2l(c,l) ({ unsigned int r=*((const unsigned int *)(c)); \ - asm ("bswapl %0":"=r"(r):"0"(r)); \ - (c)+=4; (l)=r; }) -# define HOST_l2c(l,c) ({ unsigned int r=(l); \ - asm ("bswapl %0":"=r"(r):"0"(r)); \ - *((unsigned int *)(c))=r; (c)+=4; r; }) -# endif -# endif -#endif - -#ifndef HOST_c2l -#define HOST_c2l(c,l) (l =(((unsigned long)(*((c)++)))<<24), \ - l|=(((unsigned long)(*((c)++)))<<16), \ - l|=(((unsigned long)(*((c)++)))<< 8), \ - l|=(((unsigned long)(*((c)++))) ), \ - l) -#endif -#define HOST_p_c2l(c,l,n) { \ - switch (n) { \ - case 0: l =((unsigned long)(*((c)++)))<<24; \ - case 1: l|=((unsigned long)(*((c)++)))<<16; \ - case 2: l|=((unsigned long)(*((c)++)))<< 8; \ - case 3: l|=((unsigned long)(*((c)++))); \ - } } -#define HOST_p_c2l_p(c,l,sc,len) { \ - switch (sc) { \ - case 0: l =((unsigned long)(*((c)++)))<<24; \ - if (--len == 0) break; \ - case 1: l|=((unsigned long)(*((c)++)))<<16; \ - if (--len == 0) break; \ - case 2: l|=((unsigned long)(*((c)++)))<< 8; \ - } } -/* NOTE the pointer is not incremented at the end of this */ -#define HOST_c2l_p(c,l,n) { \ - l=0; (c)+=n; \ - switch (n) { \ - case 3: l =((unsigned long)(*(--(c))))<< 8; \ - case 2: l|=((unsigned long)(*(--(c))))<<16; \ - case 1: l|=((unsigned long)(*(--(c))))<<24; \ - } } -#ifndef HOST_l2c -#define HOST_l2c(l,c) (*((c)++)=(unsigned char)(((l)>>24)&0xff), \ - *((c)++)=(unsigned char)(((l)>>16)&0xff), \ - *((c)++)=(unsigned char)(((l)>> 8)&0xff), \ - *((c)++)=(unsigned char)(((l) )&0xff), \ - l) -#endif - -#elif defined(DATA_ORDER_IS_LITTLE_ENDIAN) - -#if defined(__i386) || defined(__i386__) || defined(__x86_64) || defined(__x86_64__) - /* See comment in DATA_ORDER_IS_BIG_ENDIAN section. */ -# define HOST_c2l(c,l) ((l)=*((const unsigned int *)(c)), (c)+=4, l) -# define HOST_l2c(l,c) (*((unsigned int *)(c))=(l), (c)+=4, l) -#endif - -#ifndef HOST_c2l -#define HOST_c2l(c,l) (l =(((unsigned long)(*((c)++))) ), \ - l|=(((unsigned long)(*((c)++)))<< 8), \ - l|=(((unsigned long)(*((c)++)))<<16), \ - l|=(((unsigned long)(*((c)++)))<<24), \ - l) -#endif -#define HOST_p_c2l(c,l,n) { \ - switch (n) { \ - case 0: l =((unsigned long)(*((c)++))); \ - case 1: l|=((unsigned long)(*((c)++)))<< 8; \ - case 2: l|=((unsigned long)(*((c)++)))<<16; \ - case 3: l|=((unsigned long)(*((c)++)))<<24; \ - } } -#define HOST_p_c2l_p(c,l,sc,len) { \ - switch (sc) { \ - case 0: l =((unsigned long)(*((c)++))); \ - if (--len == 0) break; \ - case 1: l|=((unsigned long)(*((c)++)))<< 8; \ - if (--len == 0) break; \ - case 2: l|=((unsigned long)(*((c)++)))<<16; \ - } } -/* NOTE the pointer is not incremented at the end of this */ -#define HOST_c2l_p(c,l,n) { \ - l=0; (c)+=n; \ - switch (n) { \ - case 3: l =((unsigned long)(*(--(c))))<<16; \ - case 2: l|=((unsigned long)(*(--(c))))<< 8; \ - case 1: l|=((unsigned long)(*(--(c)))); \ - } } -#ifndef HOST_l2c -#define HOST_l2c(l,c) (*((c)++)=(unsigned char)(((l) )&0xff), \ - *((c)++)=(unsigned char)(((l)>> 8)&0xff), \ - *((c)++)=(unsigned char)(((l)>>16)&0xff), \ - *((c)++)=(unsigned char)(((l)>>24)&0xff), \ - l) -#endif - -#endif - -/* - * Time for some action:-) - */ - -int HASH_UPDATE (HASH_CTX *c, const void *data_, size_t len) - { - const unsigned char *data=data_; - register HASH_LONG * p; - register HASH_LONG l; - size_t sw,sc,ew,ec; - - if (len==0) return 1; - - l=(c->Nl+(((HASH_LONG)len)<<3))&0xffffffffUL; - /* 95-05-24 eay Fixed a bug with the overflow handling, thanks to - * Wei Dai <weidai@eskimo.com> for pointing it out. */ - if (l < c->Nl) /* overflow */ - c->Nh++; - c->Nh+=(len>>29); /* might cause compiler warning on 16-bit */ - c->Nl=l; - - if (c->num != 0) - { - p=c->data; - sw=c->num>>2; - sc=c->num&0x03; - - if ((c->num+len) >= HASH_CBLOCK) - { - l=p[sw]; HOST_p_c2l(data,l,sc); p[sw++]=l; - for (; sw<HASH_LBLOCK; sw++) - { - HOST_c2l(data,l); p[sw]=l; - } - HASH_BLOCK_HOST_ORDER (c,p,1); - len-=(HASH_CBLOCK-c->num); - c->num=0; - /* drop through and do the rest */ - } - else - { - c->num+=(unsigned int)len; - if ((sc+len) < 4) /* ugly, add char's to a word */ - { - l=p[sw]; HOST_p_c2l_p(data,l,sc,len); p[sw]=l; - } - else - { - ew=(c->num>>2); - ec=(c->num&0x03); - if (sc) - l=p[sw]; - HOST_p_c2l(data,l,sc); - p[sw++]=l; - for (; sw < ew; sw++) - { - HOST_c2l(data,l); p[sw]=l; - } - if (ec) - { - HOST_c2l_p(data,l,ec); p[sw]=l; - } - } - return 1; - } - } - - sw=len/HASH_CBLOCK; - if (sw > 0) - { -#if defined(HASH_BLOCK_DATA_ORDER_ALIGNED) - /* - * Note that HASH_BLOCK_DATA_ORDER_ALIGNED gets defined - * only if sizeof(HASH_LONG)==4. - */ - if ((((size_t)data)%4) == 0) - { - /* data is properly aligned so that we can cast it: */ - HASH_BLOCK_DATA_ORDER_ALIGNED (c,(const HASH_LONG *)data,sw); - sw*=HASH_CBLOCK; - data+=sw; - len-=sw; - } - else -#if !defined(HASH_BLOCK_DATA_ORDER) - while (sw--) - { - memcpy (p=c->data,data,HASH_CBLOCK); - HASH_BLOCK_DATA_ORDER_ALIGNED(c,p,1); - data+=HASH_CBLOCK; - len-=HASH_CBLOCK; - } -#endif -#endif -#if defined(HASH_BLOCK_DATA_ORDER) - { - HASH_BLOCK_DATA_ORDER(c,data,sw); - sw*=HASH_CBLOCK; - data+=sw; - len-=sw; - } -#endif - } - - if (len!=0) - { - p = c->data; - c->num = len; - ew=len>>2; /* words to copy */ - ec=len&0x03; - for (; ew; ew--,p++) - { - HOST_c2l(data,l); *p=l; - } - HOST_c2l_p(data,l,ec); - *p=l; - } - return 1; - } - - -void HASH_TRANSFORM (HASH_CTX *c, const unsigned char *data) - { -#if defined(HASH_BLOCK_DATA_ORDER_ALIGNED) - if ((((size_t)data)%4) == 0) - /* data is properly aligned so that we can cast it: */ - HASH_BLOCK_DATA_ORDER_ALIGNED (c,(const HASH_LONG *)data,1); - else -#if !defined(HASH_BLOCK_DATA_ORDER) - { - memcpy (c->data,data,HASH_CBLOCK); - HASH_BLOCK_DATA_ORDER_ALIGNED (c,c->data,1); - } -#endif -#endif -#if defined(HASH_BLOCK_DATA_ORDER) - HASH_BLOCK_DATA_ORDER (c,data,1); -#endif - } - - -int HASH_FINAL (unsigned char *md, HASH_CTX *c) - { - register HASH_LONG *p; - register unsigned long l; - register int i,j; - static const unsigned char end[4]={0x80,0x00,0x00,0x00}; - const unsigned char *cp=end; - - /* c->num should definitly have room for at least one more byte. */ - p=c->data; - i=c->num>>2; - j=c->num&0x03; - -#if 0 - /* purify often complains about the following line as an - * Uninitialized Memory Read. While this can be true, the - * following p_c2l macro will reset l when that case is true. - * This is because j&0x03 contains the number of 'valid' bytes - * already in p[i]. If and only if j&0x03 == 0, the UMR will - * occur but this is also the only time p_c2l will do - * l= *(cp++) instead of l|= *(cp++) - * Many thanks to Alex Tang <altitude@cic.net> for pickup this - * 'potential bug' */ -#ifdef PURIFY - if (j==0) p[i]=0; /* Yeah, but that's not the way to fix it:-) */ -#endif - l=p[i]; -#else - l = (j==0) ? 0 : p[i]; -#endif - HOST_p_c2l(cp,l,j); p[i++]=l; /* i is the next 'undefined word' */ - - if (i>(HASH_LBLOCK-2)) /* save room for Nl and Nh */ - { - if (i<HASH_LBLOCK) p[i]=0; - HASH_BLOCK_HOST_ORDER (c,p,1); - i=0; - } - for (; i<(HASH_LBLOCK-2); i++) - p[i]=0; - -#if defined(DATA_ORDER_IS_BIG_ENDIAN) - p[HASH_LBLOCK-2]=c->Nh; - p[HASH_LBLOCK-1]=c->Nl; -#elif defined(DATA_ORDER_IS_LITTLE_ENDIAN) - p[HASH_LBLOCK-2]=c->Nl; - p[HASH_LBLOCK-1]=c->Nh; -#endif - HASH_BLOCK_HOST_ORDER (c,p,1); - -#ifndef HASH_MAKE_STRING -#error "HASH_MAKE_STRING must be defined!" -#else - HASH_MAKE_STRING(c,md); -#endif - - c->num=0; - /* clear stuff, HASH_BLOCK may be leaving some stuff on the stack - * but I'm not worried :-) - OPENSSL_cleanse((void *)c,sizeof(HASH_CTX)); - */ - return 1; - } - -#ifndef MD32_REG_T -#define MD32_REG_T long -/* - * This comment was originaly written for MD5, which is why it - * discusses A-D. But it basically applies to all 32-bit digests, - * which is why it was moved to common header file. - * - * In case you wonder why A-D are declared as long and not - * as MD5_LONG. Doing so results in slight performance - * boost on LP64 architectures. The catch is we don't - * really care if 32 MSBs of a 64-bit register get polluted - * with eventual overflows as we *save* only 32 LSBs in - * *either* case. Now declaring 'em long excuses the compiler - * from keeping 32 MSBs zeroed resulting in 13% performance - * improvement under SPARC Solaris7/64 and 5% under AlphaLinux. - * Well, to be honest it should say that this *prevents* - * performance degradation. - * <appro@fy.chalmers.se> - * Apparently there're LP64 compilers that generate better - * code if A-D are declared int. Most notably GCC-x86_64 - * generates better code. - * <appro@fy.chalmers.se> - */ -#endif diff --git a/src/libutil/md5.c b/src/libutil/md5.c deleted file mode 100644 index b31640cdcc..0000000000 --- a/src/libutil/md5.c +++ /dev/null @@ -1,365 +0,0 @@ -/* Functions to compute MD5 message digest of files or memory blocks. - according to the definition of MD5 in RFC 1321 from April 1992. - Copyright (C) 1995,1996,1997,1999,2000,2001 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, write to the Free - Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA - 02111-1307 USA. */ - -/* Written by Ulrich Drepper <drepper@gnu.ai.mit.edu>, 1995. */ - -#include <sys/types.h> - -#include <stdlib.h> -#include <string.h> - -#include "md5.h" - - -static md5_uint32 SWAP(md5_uint32 n) -{ - static int checked = 0; - static int bigendian = 0; - static md5_uint32 test; - - if (!checked) { - test = 1; - if (* (char *) &test == 0) - bigendian = 1; - checked = 1; - } - - if (bigendian) - return (((n) << 24) | (((n) & 0xff00) << 8) | (((n) >> 8) & 0xff00) | ((n) >> 24)); - else - return n; -} - - -/* This array contains the bytes used to pad the buffer to the next - 64-byte boundary. (RFC 1321, 3.1: Step 1) */ -static const unsigned char fillbuf[64] = { 0x80, 0 /* , 0, 0, ... */ }; - - -/* Initialize structure containing state of computation. - (RFC 1321, 3.3: Step 3) */ -void -MD5_Init (ctx) - struct MD5_CTX *ctx; -{ - ctx->A = 0x67452301; - ctx->B = 0xefcdab89; - ctx->C = 0x98badcfe; - ctx->D = 0x10325476; - - ctx->total[0] = ctx->total[1] = 0; - ctx->buflen = 0; -} - -/* Put result from CTX in first 16 bytes following RESBUF. The result - must be in little endian byte order. - - IMPORTANT: On some systems it is required that RESBUF is correctly - aligned for a 32 bits value. */ -void * -md5_read_ctx (ctx, resbuf) - const struct MD5_CTX *ctx; - void *resbuf; -{ - ((md5_uint32 *) resbuf)[0] = SWAP (ctx->A); - ((md5_uint32 *) resbuf)[1] = SWAP (ctx->B); - ((md5_uint32 *) resbuf)[2] = SWAP (ctx->C); - ((md5_uint32 *) resbuf)[3] = SWAP (ctx->D); - - return resbuf; -} - -/* Process the remaining bytes in the internal buffer and the usual - prolog according to the standard and write the result to RESBUF. - - IMPORTANT: On some systems it is required that RESBUF is correctly - aligned for a 32 bits value. */ -void * -MD5_Final (resbuf, ctx) - void *resbuf; - struct MD5_CTX *ctx; -{ - /* Take yet unprocessed bytes into account. */ - md5_uint32 bytes = ctx->buflen; - size_t pad; - - /* Now count remaining bytes. */ - ctx->total[0] += bytes; - if (ctx->total[0] < bytes) - ++ctx->total[1]; - - pad = bytes >= 56 ? 64 + 56 - bytes : 56 - bytes; - memcpy (&ctx->buffer[bytes], fillbuf, pad); - - /* Put the 64-bit file length in *bits* at the end of the buffer. */ - *(md5_uint32 *) &ctx->buffer[bytes + pad] = SWAP (ctx->total[0] << 3); - *(md5_uint32 *) &ctx->buffer[bytes + pad + 4] = SWAP ((ctx->total[1] << 3) | - (ctx->total[0] >> 29)); - - /* Process last bytes. */ - md5_process_block (ctx->buffer, bytes + pad + 8, ctx); - - return md5_read_ctx (ctx, resbuf); -} - -void -MD5_Update (ctx, buffer, len) - struct MD5_CTX *ctx; - const void *buffer; - size_t len; -{ - /* When we already have some bits in our internal buffer concatenate - both inputs first. */ - if (ctx->buflen != 0) - { - size_t left_over = ctx->buflen; - size_t add = 128 - left_over > len ? len : 128 - left_over; - - memcpy (&ctx->buffer[left_over], buffer, add); - ctx->buflen += add; - - if (ctx->buflen > 64) - { - md5_process_block (ctx->buffer, ctx->buflen & ~63, ctx); - - ctx->buflen &= 63; - /* The regions in the following copy operation cannot overlap. */ - memcpy (ctx->buffer, &ctx->buffer[(left_over + add) & ~63], - ctx->buflen); - } - - buffer = (const char *) buffer + add; - len -= add; - } - - /* Process available complete blocks. */ - if (len >= 64) - { -#if !_STRING_ARCH_unaligned -/* To check alignment gcc has an appropriate operator. Other - compilers don't. */ -# if __GNUC__ >= 2 -# define UNALIGNED_P(p) (((md5_uintptr) p) % __alignof__ (md5_uint32) != 0) -# else -# define UNALIGNED_P(p) (((md5_uintptr) p) % sizeof (md5_uint32) != 0) -# endif - if (UNALIGNED_P (buffer)) - while (len > 64) - { - md5_process_block (memcpy (ctx->buffer, buffer, 64), 64, ctx); - buffer = (const char *) buffer + 64; - len -= 64; - } - else -#endif - { - md5_process_block (buffer, len & ~63, ctx); - buffer = (const char *) buffer + (len & ~63); - len &= 63; - } - } - - /* Move remaining bytes in internal buffer. */ - if (len > 0) - { - size_t left_over = ctx->buflen; - - memcpy (&ctx->buffer[left_over], buffer, len); - left_over += len; - if (left_over >= 64) - { - md5_process_block (ctx->buffer, 64, ctx); - left_over -= 64; - memcpy (ctx->buffer, &ctx->buffer[64], left_over); - } - ctx->buflen = left_over; - } -} - - -/* These are the four functions used in the four steps of the MD5 algorithm - and defined in the RFC 1321. The first function is a little bit optimized - (as found in Colin Plumbs public domain implementation). */ -/* #define FF(b, c, d) ((b & c) | (~b & d)) */ -#define FF(b, c, d) (d ^ (b & (c ^ d))) -#define FG(b, c, d) FF (d, b, c) -#define FH(b, c, d) (b ^ c ^ d) -#define FI(b, c, d) (c ^ (b | ~d)) - -/* Process LEN bytes of BUFFER, accumulating context into CTX. - It is assumed that LEN % 64 == 0. */ - -void -md5_process_block (buffer, len, ctx) - const void *buffer; - size_t len; - struct MD5_CTX *ctx; -{ - md5_uint32 correct_words[16]; - const md5_uint32 *words = buffer; - size_t nwords = len / sizeof (md5_uint32); - const md5_uint32 *endp = words + nwords; - md5_uint32 A = ctx->A; - md5_uint32 B = ctx->B; - md5_uint32 C = ctx->C; - md5_uint32 D = ctx->D; - - /* First increment the byte count. RFC 1321 specifies the possible - length of the file up to 2^64 bits. Here we only compute the - number of bytes. Do a double word increment. */ - ctx->total[0] += len; - if (ctx->total[0] < len) - ++ctx->total[1]; - - /* Process all bytes in the buffer with 64 bytes in each round of - the loop. */ - while (words < endp) - { - md5_uint32 *cwp = correct_words; - md5_uint32 A_save = A; - md5_uint32 B_save = B; - md5_uint32 C_save = C; - md5_uint32 D_save = D; - - /* First round: using the given function, the context and a constant - the next context is computed. Because the algorithms processing - unit is a 32-bit word and it is determined to work on words in - little endian byte order we perhaps have to change the byte order - before the computation. To reduce the work for the next steps - we store the swapped words in the array CORRECT_WORDS. */ - -#define OP(a, b, c, d, s, T) \ - do \ - { \ - a += FF (b, c, d) + (*cwp++ = SWAP (*words)) + T; \ - ++words; \ - CYCLIC (a, s); \ - a += b; \ - } \ - while (0) - - /* It is unfortunate that C does not provide an operator for - cyclic rotation. Hope the C compiler is smart enough. */ -#define CYCLIC(w, s) (w = (w << s) | (w >> (32 - s))) - - /* Before we start, one word to the strange constants. - They are defined in RFC 1321 as - - T[i] = (int) (4294967296.0 * fabs (sin (i))), i=1..64 - */ - - /* Round 1. */ - OP (A, B, C, D, 7, 0xd76aa478); - OP (D, A, B, C, 12, 0xe8c7b756); - OP (C, D, A, B, 17, 0x242070db); - OP (B, C, D, A, 22, 0xc1bdceee); - OP (A, B, C, D, 7, 0xf57c0faf); - OP (D, A, B, C, 12, 0x4787c62a); - OP (C, D, A, B, 17, 0xa8304613); - OP (B, C, D, A, 22, 0xfd469501); - OP (A, B, C, D, 7, 0x698098d8); - OP (D, A, B, C, 12, 0x8b44f7af); - OP (C, D, A, B, 17, 0xffff5bb1); - OP (B, C, D, A, 22, 0x895cd7be); - OP (A, B, C, D, 7, 0x6b901122); - OP (D, A, B, C, 12, 0xfd987193); - OP (C, D, A, B, 17, 0xa679438e); - OP (B, C, D, A, 22, 0x49b40821); - - /* For the second to fourth round we have the possibly swapped words - in CORRECT_WORDS. Redefine the macro to take an additional first - argument specifying the function to use. */ -#undef OP -#define OP(f, a, b, c, d, k, s, T) \ - do \ - { \ - a += f (b, c, d) + correct_words[k] + T; \ - CYCLIC (a, s); \ - a += b; \ - } \ - while (0) - - /* Round 2. */ - OP (FG, A, B, C, D, 1, 5, 0xf61e2562); - OP (FG, D, A, B, C, 6, 9, 0xc040b340); - OP (FG, C, D, A, B, 11, 14, 0x265e5a51); - OP (FG, B, C, D, A, 0, 20, 0xe9b6c7aa); - OP (FG, A, B, C, D, 5, 5, 0xd62f105d); - OP (FG, D, A, B, C, 10, 9, 0x02441453); - OP (FG, C, D, A, B, 15, 14, 0xd8a1e681); - OP (FG, B, C, D, A, 4, 20, 0xe7d3fbc8); - OP (FG, A, B, C, D, 9, 5, 0x21e1cde6); - OP (FG, D, A, B, C, 14, 9, 0xc33707d6); - OP (FG, C, D, A, B, 3, 14, 0xf4d50d87); - OP (FG, B, C, D, A, 8, 20, 0x455a14ed); - OP (FG, A, B, C, D, 13, 5, 0xa9e3e905); - OP (FG, D, A, B, C, 2, 9, 0xfcefa3f8); - OP (FG, C, D, A, B, 7, 14, 0x676f02d9); - OP (FG, B, C, D, A, 12, 20, 0x8d2a4c8a); - - /* Round 3. */ - OP (FH, A, B, C, D, 5, 4, 0xfffa3942); - OP (FH, D, A, B, C, 8, 11, 0x8771f681); - OP (FH, C, D, A, B, 11, 16, 0x6d9d6122); - OP (FH, B, C, D, A, 14, 23, 0xfde5380c); - OP (FH, A, B, C, D, 1, 4, 0xa4beea44); - OP (FH, D, A, B, C, 4, 11, 0x4bdecfa9); - OP (FH, C, D, A, B, 7, 16, 0xf6bb4b60); - OP (FH, B, C, D, A, 10, 23, 0xbebfbc70); - OP (FH, A, B, C, D, 13, 4, 0x289b7ec6); - OP (FH, D, A, B, C, 0, 11, 0xeaa127fa); - OP (FH, C, D, A, B, 3, 16, 0xd4ef3085); - OP (FH, B, C, D, A, 6, 23, 0x04881d05); - OP (FH, A, B, C, D, 9, 4, 0xd9d4d039); - OP (FH, D, A, B, C, 12, 11, 0xe6db99e5); - OP (FH, C, D, A, B, 15, 16, 0x1fa27cf8); - OP (FH, B, C, D, A, 2, 23, 0xc4ac5665); - - /* Round 4. */ - OP (FI, A, B, C, D, 0, 6, 0xf4292244); - OP (FI, D, A, B, C, 7, 10, 0x432aff97); - OP (FI, C, D, A, B, 14, 15, 0xab9423a7); - OP (FI, B, C, D, A, 5, 21, 0xfc93a039); - OP (FI, A, B, C, D, 12, 6, 0x655b59c3); - OP (FI, D, A, B, C, 3, 10, 0x8f0ccc92); - OP (FI, C, D, A, B, 10, 15, 0xffeff47d); - OP (FI, B, C, D, A, 1, 21, 0x85845dd1); - OP (FI, A, B, C, D, 8, 6, 0x6fa87e4f); - OP (FI, D, A, B, C, 15, 10, 0xfe2ce6e0); - OP (FI, C, D, A, B, 6, 15, 0xa3014314); - OP (FI, B, C, D, A, 13, 21, 0x4e0811a1); - OP (FI, A, B, C, D, 4, 6, 0xf7537e82); - OP (FI, D, A, B, C, 11, 10, 0xbd3af235); - OP (FI, C, D, A, B, 2, 15, 0x2ad7d2bb); - OP (FI, B, C, D, A, 9, 21, 0xeb86d391); - - /* Add the starting values of the context. */ - A += A_save; - B += B_save; - C += C_save; - D += D_save; - } - - /* Put checksum in context given as argument. */ - ctx->A = A; - ctx->B = B; - ctx->C = C; - ctx->D = D; -} diff --git a/src/libutil/md5.h b/src/libutil/md5.h deleted file mode 100644 index 228d497232..0000000000 --- a/src/libutil/md5.h +++ /dev/null @@ -1,82 +0,0 @@ -/* Declaration of functions and data types used for MD5 sum computing - library functions. - Copyright (C) 1995,1996,1997,1999,2000,2001 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, write to the Free - Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA - 02111-1307 USA. */ - -#ifndef _MD5_H -#define _MD5_H 1 - -#include <inttypes.h> -typedef uint32_t md5_uint32; -typedef uintptr_t md5_uintptr; - -/* Structure to save state of computation between the single steps. */ -struct MD5_CTX -{ - md5_uint32 A; - md5_uint32 B; - md5_uint32 C; - md5_uint32 D; - - md5_uint32 total[2]; - md5_uint32 buflen; - char buffer[128] __attribute__ ((__aligned__ (__alignof__ (md5_uint32)))); -}; - -/* - * The following three functions are build up the low level used in - * the functions `md5_stream' and `md5_buffer'. - */ - -/* Initialize structure containing state of computation. - (RFC 1321, 3.3: Step 3) */ -extern void MD5_Init (struct MD5_CTX *ctx); - -/* Starting with the result of former calls of this function (or the - initialization function update the context for the next LEN bytes - starting at BUFFER. - It is necessary that LEN is a multiple of 64!!! */ -extern void md5_process_block (const void *buffer, size_t len, - struct MD5_CTX *ctx); - -/* Starting with the result of former calls of this function (or the - initialization function update the context for the next LEN bytes - starting at BUFFER. - It is NOT required that LEN is a multiple of 64. */ -extern void MD5_Update (struct MD5_CTX *ctx, const void *buffer, size_t len); - -/* Process the remaining bytes in the buffer and put result from CTX - in first 16 bytes following RESBUF. The result is always in little - endian byte order, so that a byte-wise output yields to the wanted - ASCII representation of the message digest. - - IMPORTANT: On some systems it is required that RESBUF is correctly - aligned for a 32 bits value. */ -extern void *MD5_Final (void *resbuf, struct MD5_CTX *ctx); - - -/* Put result from CTX in first 16 bytes following RESBUF. The result is - always in little endian byte order, so that a byte-wise output yields - to the wanted ASCII representation of the message digest. - - IMPORTANT: On some systems it is required that RESBUF is correctly - aligned for a 32 bits value. */ -extern void *md5_read_ctx (const struct MD5_CTX *ctx, void *resbuf); - - -#endif /* md5.h */ diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc deleted file mode 100644 index 6b71f52c15..0000000000 --- a/src/libutil/serialise.cc +++ /dev/null @@ -1,259 +0,0 @@ -#include "serialise.hh" -#include "util.hh" - -#include <cstring> -#include <cerrno> - - -namespace nix { - - -BufferedSink::~BufferedSink() -{ - /* We can't call flush() here, because C++ for some insane reason - doesn't allow you to call virtual methods from a destructor. */ - assert(!bufPos); - delete[] buffer; -} - - -void BufferedSink::operator () (const unsigned char * data, size_t len) -{ - if (!buffer) buffer = new unsigned char[bufSize]; - - while (len) { - /* Optimisation: bypass the buffer if the data exceeds the - buffer size. */ - if (bufPos + len >= bufSize) { - flush(); - write(data, len); - break; - } - /* Otherwise, copy the bytes to the buffer. Flush the buffer - when it's full. */ - size_t n = bufPos + len > bufSize ? bufSize - bufPos : len; - memcpy(buffer + bufPos, data, n); - data += n; bufPos += n; len -= n; - if (bufPos == bufSize) flush(); - } -} - - -void BufferedSink::flush() -{ - if (bufPos == 0) return; - size_t n = bufPos; - bufPos = 0; // don't trigger the assert() in ~BufferedSink() - write(buffer, n); -} - - -FdSink::~FdSink() -{ - try { flush(); } catch (...) { ignoreException(); } -} - - -void FdSink::write(const unsigned char * data, size_t len) -{ - writeFull(fd, data, len); -} - - -void Source::operator () (unsigned char * data, size_t len) -{ - while (len) { - size_t n = read(data, len); - data += n; len -= n; - } -} - - -BufferedSource::~BufferedSource() -{ - delete[] buffer; -} - - -size_t BufferedSource::read(unsigned char * data, size_t len) -{ - if (!buffer) buffer = new unsigned char[bufSize]; - - if (!bufPosIn) bufPosIn = readUnbuffered(buffer, bufSize); - - /* Copy out the data in the buffer. */ - size_t n = len > bufPosIn - bufPosOut ? bufPosIn - bufPosOut : len; - memcpy(data, buffer + bufPosOut, n); - bufPosOut += n; - if (bufPosIn == bufPosOut) bufPosIn = bufPosOut = 0; - return n; -} - - -bool BufferedSource::hasData() -{ - return bufPosOut < bufPosIn; -} - - -size_t FdSource::readUnbuffered(unsigned char * data, size_t len) -{ - ssize_t n; - do { - checkInterrupt(); - n = ::read(fd, (char *) data, bufSize); - } while (n == -1 && errno == EINTR); - if (n == -1) throw SysError("reading from file"); - if (n == 0) throw EndOfFile("unexpected end-of-file"); - return n; -} - - -size_t StringSource::read(unsigned char * data, size_t len) -{ - if (pos == s.size()) throw EndOfFile("end of string reached"); - size_t n = s.copy((char *) data, len, pos); - pos += n; - return n; -} - - -void writePadding(size_t len, Sink & sink) -{ - if (len % 8) { - unsigned char zero[8]; - memset(zero, 0, sizeof(zero)); - sink(zero, 8 - (len % 8)); - } -} - - -void writeInt(unsigned int n, Sink & sink) -{ - unsigned char buf[8]; - memset(buf, 0, sizeof(buf)); - buf[0] = n & 0xff; - buf[1] = (n >> 8) & 0xff; - buf[2] = (n >> 16) & 0xff; - buf[3] = (n >> 24) & 0xff; - sink(buf, sizeof(buf)); -} - - -void writeLongLong(unsigned long long n, Sink & sink) -{ - unsigned char buf[8]; - buf[0] = n & 0xff; - buf[1] = (n >> 8) & 0xff; - buf[2] = (n >> 16) & 0xff; - buf[3] = (n >> 24) & 0xff; - buf[4] = (n >> 32) & 0xff; - buf[5] = (n >> 40) & 0xff; - buf[6] = (n >> 48) & 0xff; - buf[7] = (n >> 56) & 0xff; - sink(buf, sizeof(buf)); -} - - -void writeString(const unsigned char * buf, size_t len, Sink & sink) -{ - writeInt(len, sink); - sink(buf, len); - writePadding(len, sink); -} - - -void writeString(const string & s, Sink & sink) -{ - writeString((const unsigned char *) s.data(), s.size(), sink); -} - - -template<class T> void writeStrings(const T & ss, Sink & sink) -{ - writeInt(ss.size(), sink); - foreach (typename T::const_iterator, i, ss) - writeString(*i, sink); -} - -template void writeStrings(const Paths & ss, Sink & sink); -template void writeStrings(const PathSet & ss, Sink & sink); - - -void readPadding(size_t len, Source & source) -{ - if (len % 8) { - unsigned char zero[8]; - size_t n = 8 - (len % 8); - source(zero, n); - for (unsigned int i = 0; i < n; i++) - if (zero[i]) throw SerialisationError("non-zero padding"); - } -} - - -unsigned int readInt(Source & source) -{ - unsigned char buf[8]; - source(buf, sizeof(buf)); - if (buf[4] || buf[5] || buf[6] || buf[7]) - throw SerialisationError("implementation cannot deal with > 32-bit integers"); - return - buf[0] | - (buf[1] << 8) | - (buf[2] << 16) | - (buf[3] << 24); -} - - -unsigned long long readLongLong(Source & source) -{ - unsigned char buf[8]; - source(buf, sizeof(buf)); - return - ((unsigned long long) buf[0]) | - ((unsigned long long) buf[1] << 8) | - ((unsigned long long) buf[2] << 16) | - ((unsigned long long) buf[3] << 24) | - ((unsigned long long) buf[4] << 32) | - ((unsigned long long) buf[5] << 40) | - ((unsigned long long) buf[6] << 48) | - ((unsigned long long) buf[7] << 56); -} - - -size_t readString(unsigned char * buf, size_t max, Source & source) -{ - size_t len = readInt(source); - if (len > max) throw Error("string is too long"); - source(buf, len); - readPadding(len, source); - return len; -} - - -string readString(Source & source) -{ - size_t len = readInt(source); - unsigned char * buf = new unsigned char[len]; - AutoDeleteArray<unsigned char> d(buf); - source(buf, len); - readPadding(len, source); - return string((char *) buf, len); -} - - -template<class T> T readStrings(Source & source) -{ - unsigned int count = readInt(source); - T ss; - while (count--) - ss.insert(ss.end(), readString(source)); - return ss; -} - -template Paths readStrings(Source & source); -template PathSet readStrings(Source & source); - - -} diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh deleted file mode 100644 index e5a9df1d05..0000000000 --- a/src/libutil/serialise.hh +++ /dev/null @@ -1,133 +0,0 @@ -#pragma once - -#include "types.hh" - - -namespace nix { - - -/* Abstract destination of binary data. */ -struct Sink -{ - virtual ~Sink() { } - virtual void operator () (const unsigned char * data, size_t len) = 0; -}; - - -/* A buffered abstract sink. */ -struct BufferedSink : Sink -{ - size_t bufSize, bufPos; - unsigned char * buffer; - - BufferedSink(size_t bufSize = 32 * 1024) - : bufSize(bufSize), bufPos(0), buffer(0) { } - ~BufferedSink(); - - void operator () (const unsigned char * data, size_t len); - - void flush(); - - virtual void write(const unsigned char * data, size_t len) = 0; -}; - - -/* Abstract source of binary data. */ -struct Source -{ - virtual ~Source() { } - - /* Store exactly âlenâ bytes in the buffer pointed to by âdataâ. - It blocks until all the requested data is available, or throws - an error if it is not going to be available. */ - void operator () (unsigned char * data, size_t len); - - /* Store up to âlenâ in the buffer pointed to by âdataâ, and - return the number of bytes stored. If blocks until at least - one byte is available. */ - virtual size_t read(unsigned char * data, size_t len) = 0; -}; - - -/* A buffered abstract source. */ -struct BufferedSource : Source -{ - size_t bufSize, bufPosIn, bufPosOut; - unsigned char * buffer; - - BufferedSource(size_t bufSize = 32 * 1024) - : bufSize(bufSize), bufPosIn(0), bufPosOut(0), buffer(0) { } - ~BufferedSource(); - - size_t read(unsigned char * data, size_t len); - - /* Underlying read call, to be overridden. */ - virtual size_t readUnbuffered(unsigned char * data, size_t len) = 0; - - bool hasData(); -}; - - -/* A sink that writes data to a file descriptor. */ -struct FdSink : BufferedSink -{ - int fd; - - FdSink() : fd(-1) { } - FdSink(int fd) : fd(fd) { } - ~FdSink(); - - void write(const unsigned char * data, size_t len); -}; - - -/* A source that reads data from a file descriptor. */ -struct FdSource : BufferedSource -{ - int fd; - FdSource() : fd(-1) { } - FdSource(int fd) : fd(fd) { } - size_t readUnbuffered(unsigned char * data, size_t len); -}; - - -/* A sink that writes data to a string. */ -struct StringSink : Sink -{ - string s; - void operator () (const unsigned char * data, size_t len) - { - s.append((const char *) data, len); - } -}; - - -/* A source that reads data from a string. */ -struct StringSource : Source -{ - const string & s; - size_t pos; - StringSource(const string & _s) : s(_s), pos(0) { } - size_t read(unsigned char * data, size_t len); -}; - - -void writePadding(size_t len, Sink & sink); -void writeInt(unsigned int n, Sink & sink); -void writeLongLong(unsigned long long n, Sink & sink); -void writeString(const unsigned char * buf, size_t len, Sink & sink); -void writeString(const string & s, Sink & sink); -template<class T> void writeStrings(const T & ss, Sink & sink); - -void readPadding(size_t len, Source & source); -unsigned int readInt(Source & source); -unsigned long long readLongLong(Source & source); -size_t readString(unsigned char * buf, size_t max, Source & source); -string readString(Source & source); -template<class T> T readStrings(Source & source); - - -MakeError(SerialisationError, Error) - - -} diff --git a/src/libutil/sha1.c b/src/libutil/sha1.c deleted file mode 100644 index d9d294d155..0000000000 --- a/src/libutil/sha1.c +++ /dev/null @@ -1,369 +0,0 @@ -/* $Id$ */ - -/* sha.c - Implementation of the Secure Hash Algorithm - * - * Copyright (C) 1995, A.M. Kuchling - * - * Distribute and use freely; there are no restrictions on further - * dissemination and usage except those imposed by the laws of your - * country of residence. - * - * Adapted to pike and some cleanup by Niels Möller. - */ - -/* $Id$ */ - -/* SHA: NIST's Secure Hash Algorithm */ - -/* Based on SHA code originally posted to sci.crypt by Peter Gutmann - in message <30ajo5$oe8@ccu2.auckland.ac.nz>. - Modified to test for endianness on creation of SHA objects by AMK. - Also, the original specification of SHA was found to have a weakness - by NSA/NIST. This code implements the fixed version of SHA. -*/ - -/* Here's the first paragraph of Peter Gutmann's posting: - -The following is my SHA (FIPS 180) code updated to allow use of the "fixed" -SHA, thanks to Jim Gillogly and an anonymous contributor for the information on -what's changed in the new version. The fix is a simple change which involves -adding a single rotate in the initial expansion function. It is unknown -whether this is an optimal solution to the problem which was discovered in the -SHA or whether it's simply a bandaid which fixes the problem with a minimum of -effort (for example the reengineering of a great many Capstone chips). -*/ - -#include "sha1.h" - -#include <string.h> - -void sha_copy(struct SHA_CTX *dest, struct SHA_CTX *src) -{ - unsigned int i; - - dest->count_l=src->count_l; - dest->count_h=src->count_h; - for(i=0; i<SHA_DIGESTLEN; i++) - dest->digest[i]=src->digest[i]; - for(i=0; i < src->index; i++) - dest->block[i] = src->block[i]; - dest->index = src->index; -} - - -/* The SHA f()-functions. The f1 and f3 functions can be optimized to - save one boolean operation each - thanks to Rich Schroeppel, - rcs@cs.arizona.edu for discovering this */ - -/*#define f1(x,y,z) ( ( x & y ) | ( ~x & z ) ) // Rounds 0-19 */ -#define f1(x,y,z) ( z ^ ( x & ( y ^ z ) ) ) /* Rounds 0-19 */ -#define f2(x,y,z) ( x ^ y ^ z ) /* Rounds 20-39 */ -/*#define f3(x,y,z) ( ( x & y ) | ( x & z ) | ( y & z ) ) // Rounds 40-59 */ -#define f3(x,y,z) ( ( x & y ) | ( z & ( x | y ) ) ) /* Rounds 40-59 */ -#define f4(x,y,z) ( x ^ y ^ z ) /* Rounds 60-79 */ - -/* The SHA Mysterious Constants */ - -#define K1 0x5A827999L /* Rounds 0-19 */ -#define K2 0x6ED9EBA1L /* Rounds 20-39 */ -#define K3 0x8F1BBCDCL /* Rounds 40-59 */ -#define K4 0xCA62C1D6L /* Rounds 60-79 */ - -/* SHA initial values */ - -#define h0init 0x67452301L -#define h1init 0xEFCDAB89L -#define h2init 0x98BADCFEL -#define h3init 0x10325476L -#define h4init 0xC3D2E1F0L - -/* 32-bit rotate left - kludged with shifts */ - -#define ROTL(n,X) ( ( (X) << (n) ) | ( (X) >> ( 32 - (n) ) ) ) - -/* The initial expanding function. The hash function is defined over an - 80-word expanded input array W, where the first 16 are copies of the input - data, and the remaining 64 are defined by - - W[ i ] = W[ i - 16 ] ^ W[ i - 14 ] ^ W[ i - 8 ] ^ W[ i - 3 ] - - This implementation generates these values on the fly in a circular - buffer - thanks to Colin Plumb, colin@nyx10.cs.du.edu for this - optimization. - - The updated SHA changes the expanding function by adding a rotate of 1 - bit. Thanks to Jim Gillogly, jim@rand.org, and an anonymous contributor - for this information */ - -#define expand(W,i) ( W[ i & 15 ] = \ - ROTL( 1, ( W[ i & 15 ] ^ W[ (i - 14) & 15 ] ^ \ - W[ (i - 8) & 15 ] ^ W[ (i - 3) & 15 ] ) ) ) - - -/* The prototype SHA sub-round. The fundamental sub-round is: - - a' = e + ROTL( 5, a ) + f( b, c, d ) + k + data; - b' = a; - c' = ROTL( 30, b ); - d' = c; - e' = d; - - but this is implemented by unrolling the loop 5 times and renaming the - variables ( e, a, b, c, d ) = ( a', b', c', d', e' ) each iteration. - This code is then replicated 20 times for each of the 4 functions, using - the next 20 values from the W[] array each time */ - -#define subRound(a, b, c, d, e, f, k, data) \ - ( e += ROTL( 5, a ) + f( b, c, d ) + k + data, b = ROTL( 30, b ) ) - -/* Initialize the SHA values */ - -void SHA1_Init(struct SHA_CTX *ctx) -{ - /* Set the h-vars to their initial values */ - ctx->digest[ 0 ] = h0init; - ctx->digest[ 1 ] = h1init; - ctx->digest[ 2 ] = h2init; - ctx->digest[ 3 ] = h3init; - ctx->digest[ 4 ] = h4init; - - /* Initialize bit count */ - ctx->count_l = ctx->count_h = 0; - - /* Initialize buffer */ - ctx->index = 0; -} - -/* Perform the SHA transformation. Note that this code, like MD5, seems to - break some optimizing compilers due to the complexity of the expressions - and the size of the basic block. It may be necessary to split it into - sections, e.g. based on the four subrounds - - Note that this function destroys the data area */ - -static void sha_transform(struct SHA_CTX *ctx, uint32_t *data ) -{ - uint32_t A, B, C, D, E; /* Local vars */ - - /* Set up first buffer and local data buffer */ - A = ctx->digest[0]; - B = ctx->digest[1]; - C = ctx->digest[2]; - D = ctx->digest[3]; - E = ctx->digest[4]; - - /* Heavy mangling, in 4 sub-rounds of 20 interations each. */ - subRound( A, B, C, D, E, f1, K1, data[ 0] ); - subRound( E, A, B, C, D, f1, K1, data[ 1] ); - subRound( D, E, A, B, C, f1, K1, data[ 2] ); - subRound( C, D, E, A, B, f1, K1, data[ 3] ); - subRound( B, C, D, E, A, f1, K1, data[ 4] ); - subRound( A, B, C, D, E, f1, K1, data[ 5] ); - subRound( E, A, B, C, D, f1, K1, data[ 6] ); - subRound( D, E, A, B, C, f1, K1, data[ 7] ); - subRound( C, D, E, A, B, f1, K1, data[ 8] ); - subRound( B, C, D, E, A, f1, K1, data[ 9] ); - subRound( A, B, C, D, E, f1, K1, data[10] ); - subRound( E, A, B, C, D, f1, K1, data[11] ); - subRound( D, E, A, B, C, f1, K1, data[12] ); - subRound( C, D, E, A, B, f1, K1, data[13] ); - subRound( B, C, D, E, A, f1, K1, data[14] ); - subRound( A, B, C, D, E, f1, K1, data[15] ); - subRound( E, A, B, C, D, f1, K1, expand( data, 16 ) ); - subRound( D, E, A, B, C, f1, K1, expand( data, 17 ) ); - subRound( C, D, E, A, B, f1, K1, expand( data, 18 ) ); - subRound( B, C, D, E, A, f1, K1, expand( data, 19 ) ); - - subRound( A, B, C, D, E, f2, K2, expand( data, 20 ) ); - subRound( E, A, B, C, D, f2, K2, expand( data, 21 ) ); - subRound( D, E, A, B, C, f2, K2, expand( data, 22 ) ); - subRound( C, D, E, A, B, f2, K2, expand( data, 23 ) ); - subRound( B, C, D, E, A, f2, K2, expand( data, 24 ) ); - subRound( A, B, C, D, E, f2, K2, expand( data, 25 ) ); - subRound( E, A, B, C, D, f2, K2, expand( data, 26 ) ); - subRound( D, E, A, B, C, f2, K2, expand( data, 27 ) ); - subRound( C, D, E, A, B, f2, K2, expand( data, 28 ) ); - subRound( B, C, D, E, A, f2, K2, expand( data, 29 ) ); - subRound( A, B, C, D, E, f2, K2, expand( data, 30 ) ); - subRound( E, A, B, C, D, f2, K2, expand( data, 31 ) ); - subRound( D, E, A, B, C, f2, K2, expand( data, 32 ) ); - subRound( C, D, E, A, B, f2, K2, expand( data, 33 ) ); - subRound( B, C, D, E, A, f2, K2, expand( data, 34 ) ); - subRound( A, B, C, D, E, f2, K2, expand( data, 35 ) ); - subRound( E, A, B, C, D, f2, K2, expand( data, 36 ) ); - subRound( D, E, A, B, C, f2, K2, expand( data, 37 ) ); - subRound( C, D, E, A, B, f2, K2, expand( data, 38 ) ); - subRound( B, C, D, E, A, f2, K2, expand( data, 39 ) ); - - subRound( A, B, C, D, E, f3, K3, expand( data, 40 ) ); - subRound( E, A, B, C, D, f3, K3, expand( data, 41 ) ); - subRound( D, E, A, B, C, f3, K3, expand( data, 42 ) ); - subRound( C, D, E, A, B, f3, K3, expand( data, 43 ) ); - subRound( B, C, D, E, A, f3, K3, expand( data, 44 ) ); - subRound( A, B, C, D, E, f3, K3, expand( data, 45 ) ); - subRound( E, A, B, C, D, f3, K3, expand( data, 46 ) ); - subRound( D, E, A, B, C, f3, K3, expand( data, 47 ) ); - subRound( C, D, E, A, B, f3, K3, expand( data, 48 ) ); - subRound( B, C, D, E, A, f3, K3, expand( data, 49 ) ); - subRound( A, B, C, D, E, f3, K3, expand( data, 50 ) ); - subRound( E, A, B, C, D, f3, K3, expand( data, 51 ) ); - subRound( D, E, A, B, C, f3, K3, expand( data, 52 ) ); - subRound( C, D, E, A, B, f3, K3, expand( data, 53 ) ); - subRound( B, C, D, E, A, f3, K3, expand( data, 54 ) ); - subRound( A, B, C, D, E, f3, K3, expand( data, 55 ) ); - subRound( E, A, B, C, D, f3, K3, expand( data, 56 ) ); - subRound( D, E, A, B, C, f3, K3, expand( data, 57 ) ); - subRound( C, D, E, A, B, f3, K3, expand( data, 58 ) ); - subRound( B, C, D, E, A, f3, K3, expand( data, 59 ) ); - - subRound( A, B, C, D, E, f4, K4, expand( data, 60 ) ); - subRound( E, A, B, C, D, f4, K4, expand( data, 61 ) ); - subRound( D, E, A, B, C, f4, K4, expand( data, 62 ) ); - subRound( C, D, E, A, B, f4, K4, expand( data, 63 ) ); - subRound( B, C, D, E, A, f4, K4, expand( data, 64 ) ); - subRound( A, B, C, D, E, f4, K4, expand( data, 65 ) ); - subRound( E, A, B, C, D, f4, K4, expand( data, 66 ) ); - subRound( D, E, A, B, C, f4, K4, expand( data, 67 ) ); - subRound( C, D, E, A, B, f4, K4, expand( data, 68 ) ); - subRound( B, C, D, E, A, f4, K4, expand( data, 69 ) ); - subRound( A, B, C, D, E, f4, K4, expand( data, 70 ) ); - subRound( E, A, B, C, D, f4, K4, expand( data, 71 ) ); - subRound( D, E, A, B, C, f4, K4, expand( data, 72 ) ); - subRound( C, D, E, A, B, f4, K4, expand( data, 73 ) ); - subRound( B, C, D, E, A, f4, K4, expand( data, 74 ) ); - subRound( A, B, C, D, E, f4, K4, expand( data, 75 ) ); - subRound( E, A, B, C, D, f4, K4, expand( data, 76 ) ); - subRound( D, E, A, B, C, f4, K4, expand( data, 77 ) ); - subRound( C, D, E, A, B, f4, K4, expand( data, 78 ) ); - subRound( B, C, D, E, A, f4, K4, expand( data, 79 ) ); - - /* Build message digest */ - ctx->digest[0] += A; - ctx->digest[1] += B; - ctx->digest[2] += C; - ctx->digest[3] += D; - ctx->digest[4] += E; -} - -#if 1 - -#ifndef EXTRACT_UCHAR -#define EXTRACT_UCHAR(p) (*(unsigned char *)(p)) -#endif - -#define STRING2INT(s) ((((((EXTRACT_UCHAR(s) << 8) \ - | EXTRACT_UCHAR(s+1)) << 8) \ - | EXTRACT_UCHAR(s+2)) << 8) \ - | EXTRACT_UCHAR(s+3)) -#else -uint32_t STRING2INT(unsigned char *s) -{ - uint32_t r; - unsigned int i; - - for (i = 0, r = 0; i < 4; i++, s++) - r = (r << 8) | *s; - return r; -} -#endif - -static void sha_block(struct SHA_CTX *ctx, const unsigned char *block) -{ - uint32_t data[SHA_DATALEN]; - unsigned int i; - - /* Update block count */ - if (!++ctx->count_l) - ++ctx->count_h; - - /* Endian independent conversion */ - for (i = 0; i<SHA_DATALEN; i++, block += 4) - data[i] = STRING2INT(block); - - sha_transform(ctx, data); -} - -void SHA1_Update(struct SHA_CTX *ctx, const unsigned char *buffer, uint32_t len) -{ - if (ctx->index) - { /* Try to fill partial block */ - unsigned left = SHA_DATASIZE - ctx->index; - if (len < left) - { - memcpy(ctx->block + ctx->index, buffer, len); - ctx->index += len; - return; /* Finished */ - } - else - { - memcpy(ctx->block + ctx->index, buffer, left); - sha_block(ctx, ctx->block); - buffer += left; - len -= left; - } - } - while (len >= SHA_DATASIZE) - { - sha_block(ctx, buffer); - buffer += SHA_DATASIZE; - len -= SHA_DATASIZE; - } - if ((ctx->index = len)) /* This assignment is intended */ - /* Buffer leftovers */ - memcpy(ctx->block, buffer, len); -} - -/* Final wrapup - pad to SHA_DATASIZE-byte boundary with the bit pattern - 1 0* (64-bit count of bits processed, MSB-first) */ - -void SHA1_Final(unsigned char *s, struct SHA_CTX *ctx) -{ - uint32_t data[SHA_DATALEN]; - unsigned int i; - unsigned int words; - - i = ctx->index; - /* Set the first char of padding to 0x80. This is safe since there is - always at least one byte free */ - ctx->block[i++] = 0x80; - - /* Fill rest of word */ - for( ; i & 3; i++) - ctx->block[i] = 0; - - /* i is now a multiple of the word size 4 */ - words = i >> 2; - for (i = 0; i < words; i++) - data[i] = STRING2INT(ctx->block + 4*i); - - if (words > (SHA_DATALEN-2)) - { /* No room for length in this block. Process it and - * pad with another one */ - for (i = words ; i < SHA_DATALEN; i++) - data[i] = 0; - sha_transform(ctx, data); - for (i = 0; i < (SHA_DATALEN-2); i++) - data[i] = 0; - } - else - for (i = words ; i < SHA_DATALEN - 2; i++) - data[i] = 0; - /* Theres 512 = 2^9 bits in one block */ - data[SHA_DATALEN-2] = (ctx->count_h << 9) | (ctx->count_l >> 23); - data[SHA_DATALEN-1] = (ctx->count_l << 9) | (ctx->index << 3); - sha_transform(ctx, data); - sha_digest(ctx, s); -} - -void sha_digest(struct SHA_CTX *ctx, unsigned char *s) -{ - unsigned int i; - - for (i = 0; i < SHA_DIGESTLEN; i++) - { - *s++ = ctx->digest[i] >> 24; - *s++ = 0xff & (ctx->digest[i] >> 16); - *s++ = 0xff & (ctx->digest[i] >> 8); - *s++ = 0xff & ctx->digest[i]; - } -} diff --git a/src/libutil/sha1.h b/src/libutil/sha1.h deleted file mode 100644 index 715040dd48..0000000000 --- a/src/libutil/sha1.h +++ /dev/null @@ -1,28 +0,0 @@ -#ifndef _SHA_H -#define _SHA_H - -#include <inttypes.h> - -/* The SHA block size and message digest sizes, in bytes */ - -#define SHA_DATASIZE 64 -#define SHA_DATALEN 16 -#define SHA_DIGESTSIZE 20 -#define SHA_DIGESTLEN 5 -/* The structure for storing SHA info */ - -struct SHA_CTX { - uint32_t digest[SHA_DIGESTLEN]; /* Message digest */ - uint32_t count_l, count_h; /* 64-bit block count */ - uint8_t block[SHA_DATASIZE]; /* SHA data buffer */ - unsigned int index; /* index into buffer */ -}; - -void SHA1_Init(struct SHA_CTX *ctx); -void SHA1_Update(struct SHA_CTX *ctx, const unsigned char *buffer, uint32_t len); -void SHA1_Final(unsigned char *s, struct SHA_CTX *ctx); -void sha_digest(struct SHA_CTX *ctx, unsigned char *s); -void sha_copy(struct SHA_CTX *dest, struct SHA_CTX *src); - - -#endif /* !_SHA_H */ diff --git a/src/libutil/sha256.c b/src/libutil/sha256.c deleted file mode 100644 index 63ed0ba430..0000000000 --- a/src/libutil/sha256.c +++ /dev/null @@ -1,238 +0,0 @@ -/* crypto/sha/sha256.c */ -/* ==================================================================== - * Copyright (c) 2004 The OpenSSL Project. All rights reserved - * according to the OpenSSL license [found in ./md32_common.h]. - * ==================================================================== - */ - -#include <stdlib.h> -#include <string.h> - -#include "sha256.h" - -int SHA224_Init (SHA256_CTX *c) - { - c->h[0]=0xc1059ed8UL; c->h[1]=0x367cd507UL; - c->h[2]=0x3070dd17UL; c->h[3]=0xf70e5939UL; - c->h[4]=0xffc00b31UL; c->h[5]=0x68581511UL; - c->h[6]=0x64f98fa7UL; c->h[7]=0xbefa4fa4UL; - c->Nl=0; c->Nh=0; - c->num=0; c->md_len=SHA224_DIGEST_LENGTH; - return 1; - } - -int SHA256_Init (SHA256_CTX *c) - { - c->h[0]=0x6a09e667UL; c->h[1]=0xbb67ae85UL; - c->h[2]=0x3c6ef372UL; c->h[3]=0xa54ff53aUL; - c->h[4]=0x510e527fUL; c->h[5]=0x9b05688cUL; - c->h[6]=0x1f83d9abUL; c->h[7]=0x5be0cd19UL; - c->Nl=0; c->Nh=0; - c->num=0; c->md_len=SHA256_DIGEST_LENGTH; - return 1; - } - -unsigned char *SHA224(const unsigned char *d, size_t n, unsigned char *md) - { - SHA256_CTX c; - static unsigned char m[SHA224_DIGEST_LENGTH]; - - if (md == NULL) md=m; - SHA224_Init(&c); - SHA256_Update(&c,d,n); - SHA256_Final(md,&c); - return(md); - } - -unsigned char *SHA256(const unsigned char *d, size_t n, unsigned char *md) - { - SHA256_CTX c; - static unsigned char m[SHA256_DIGEST_LENGTH]; - - if (md == NULL) md=m; - SHA256_Init(&c); - SHA256_Update(&c,d,n); - SHA256_Final(md,&c); - return(md); - } - -int SHA224_Update(SHA256_CTX *c, const void *data, size_t len) -{ return SHA256_Update (c,data,len); } -int SHA224_Final (unsigned char *md, SHA256_CTX *c) -{ return SHA256_Final (md,c); } - -#define DATA_ORDER_IS_BIG_ENDIAN - -#define HASH_LONG uint32_t -#define HASH_LONG_LOG2 2 -#define HASH_CTX SHA256_CTX -#define HASH_CBLOCK SHA_CBLOCK -#define HASH_LBLOCK SHA_LBLOCK -/* - * Note that FIPS180-2 discusses "Truncation of the Hash Function Output." - * default: case below covers for it. It's not clear however if it's - * permitted to truncate to amount of bytes not divisible by 4. I bet not, - * but if it is, then default: case shall be extended. For reference. - * Idea behind separate cases for pre-defined lenghts is to let the - * compiler decide if it's appropriate to unroll small loops. - */ -#define HASH_MAKE_STRING(c,s) do { \ - unsigned long ll; \ - unsigned int n; \ - switch ((c)->md_len) \ - { case SHA224_DIGEST_LENGTH: \ - for (n=0;n<SHA224_DIGEST_LENGTH/4;n++) \ - { ll=(c)->h[n]; HOST_l2c(ll,(s)); } \ - break; \ - case SHA256_DIGEST_LENGTH: \ - for (n=0;n<SHA256_DIGEST_LENGTH/4;n++) \ - { ll=(c)->h[n]; HOST_l2c(ll,(s)); } \ - break; \ - default: \ - if ((c)->md_len > SHA256_DIGEST_LENGTH) \ - return 0; \ - for (n=0;n<(c)->md_len/4;n++) \ - { ll=(c)->h[n]; HOST_l2c(ll,(s)); } \ - break; \ - } \ - } while (0) - -#define HASH_UPDATE SHA256_Update -#define HASH_TRANSFORM SHA256_Transform -#define HASH_FINAL SHA256_Final -#define HASH_BLOCK_HOST_ORDER sha256_block_host_order -#define HASH_BLOCK_DATA_ORDER sha256_block_data_order -void sha256_block_host_order (SHA256_CTX *ctx, const void *in, size_t num); -void sha256_block_data_order (SHA256_CTX *ctx, const void *in, size_t num); - -#include "md32_common.h" - -static const uint32_t K256[64] = { - 0x428a2f98UL,0x71374491UL,0xb5c0fbcfUL,0xe9b5dba5UL, - 0x3956c25bUL,0x59f111f1UL,0x923f82a4UL,0xab1c5ed5UL, - 0xd807aa98UL,0x12835b01UL,0x243185beUL,0x550c7dc3UL, - 0x72be5d74UL,0x80deb1feUL,0x9bdc06a7UL,0xc19bf174UL, - 0xe49b69c1UL,0xefbe4786UL,0x0fc19dc6UL,0x240ca1ccUL, - 0x2de92c6fUL,0x4a7484aaUL,0x5cb0a9dcUL,0x76f988daUL, - 0x983e5152UL,0xa831c66dUL,0xb00327c8UL,0xbf597fc7UL, - 0xc6e00bf3UL,0xd5a79147UL,0x06ca6351UL,0x14292967UL, - 0x27b70a85UL,0x2e1b2138UL,0x4d2c6dfcUL,0x53380d13UL, - 0x650a7354UL,0x766a0abbUL,0x81c2c92eUL,0x92722c85UL, - 0xa2bfe8a1UL,0xa81a664bUL,0xc24b8b70UL,0xc76c51a3UL, - 0xd192e819UL,0xd6990624UL,0xf40e3585UL,0x106aa070UL, - 0x19a4c116UL,0x1e376c08UL,0x2748774cUL,0x34b0bcb5UL, - 0x391c0cb3UL,0x4ed8aa4aUL,0x5b9cca4fUL,0x682e6ff3UL, - 0x748f82eeUL,0x78a5636fUL,0x84c87814UL,0x8cc70208UL, - 0x90befffaUL,0xa4506cebUL,0xbef9a3f7UL,0xc67178f2UL }; - -/* - * FIPS specification refers to right rotations, while our ROTATE macro - * is left one. This is why you might notice that rotation coefficients - * differ from those observed in FIPS document by 32-N... - */ -#define Sigma0(x) (ROTATE((x),30) ^ ROTATE((x),19) ^ ROTATE((x),10)) -#define Sigma1(x) (ROTATE((x),26) ^ ROTATE((x),21) ^ ROTATE((x),7)) -#define sigma0(x) (ROTATE((x),25) ^ ROTATE((x),14) ^ ((x)>>3)) -#define sigma1(x) (ROTATE((x),15) ^ ROTATE((x),13) ^ ((x)>>10)) - -#define Ch(x,y,z) (((x) & (y)) ^ ((~(x)) & (z))) -#define Maj(x,y,z) (((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z))) - -#define ROUND_00_15(i,a,b,c,d,e,f,g,h) do { \ - T1 += h + Sigma1(e) + Ch(e,f,g) + K256[i]; \ - h = Sigma0(a) + Maj(a,b,c); \ - d += T1; h += T1; } while (0) - -#define ROUND_16_63(i,a,b,c,d,e,f,g,h,X) do { \ - s0 = X[(i+1)&0x0f]; s0 = sigma0(s0); \ - s1 = X[(i+14)&0x0f]; s1 = sigma1(s1); \ - T1 = X[(i)&0x0f] += s0 + s1 + X[(i+9)&0x0f]; \ - ROUND_00_15(i,a,b,c,d,e,f,g,h); } while (0) - -static void sha256_block (SHA256_CTX *ctx, const void *in, size_t num, int host) - { - uint32_t a,b,c,d,e,f,g,h,s0,s1,T1; - uint32_t X[16]; - int i; - const unsigned char *data=in; - - while (num--) { - - a = ctx->h[0]; b = ctx->h[1]; c = ctx->h[2]; d = ctx->h[3]; - e = ctx->h[4]; f = ctx->h[5]; g = ctx->h[6]; h = ctx->h[7]; - - if (host) - { - const uint32_t *W=(const uint32_t *)data; - - T1 = X[0] = W[0]; ROUND_00_15(0,a,b,c,d,e,f,g,h); - T1 = X[1] = W[1]; ROUND_00_15(1,h,a,b,c,d,e,f,g); - T1 = X[2] = W[2]; ROUND_00_15(2,g,h,a,b,c,d,e,f); - T1 = X[3] = W[3]; ROUND_00_15(3,f,g,h,a,b,c,d,e); - T1 = X[4] = W[4]; ROUND_00_15(4,e,f,g,h,a,b,c,d); - T1 = X[5] = W[5]; ROUND_00_15(5,d,e,f,g,h,a,b,c); - T1 = X[6] = W[6]; ROUND_00_15(6,c,d,e,f,g,h,a,b); - T1 = X[7] = W[7]; ROUND_00_15(7,b,c,d,e,f,g,h,a); - T1 = X[8] = W[8]; ROUND_00_15(8,a,b,c,d,e,f,g,h); - T1 = X[9] = W[9]; ROUND_00_15(9,h,a,b,c,d,e,f,g); - T1 = X[10] = W[10]; ROUND_00_15(10,g,h,a,b,c,d,e,f); - T1 = X[11] = W[11]; ROUND_00_15(11,f,g,h,a,b,c,d,e); - T1 = X[12] = W[12]; ROUND_00_15(12,e,f,g,h,a,b,c,d); - T1 = X[13] = W[13]; ROUND_00_15(13,d,e,f,g,h,a,b,c); - T1 = X[14] = W[14]; ROUND_00_15(14,c,d,e,f,g,h,a,b); - T1 = X[15] = W[15]; ROUND_00_15(15,b,c,d,e,f,g,h,a); - - data += SHA256_CBLOCK; - } - else - { - uint32_t l; - - HOST_c2l(data,l); T1 = X[0] = l; ROUND_00_15(0,a,b,c,d,e,f,g,h); - HOST_c2l(data,l); T1 = X[1] = l; ROUND_00_15(1,h,a,b,c,d,e,f,g); - HOST_c2l(data,l); T1 = X[2] = l; ROUND_00_15(2,g,h,a,b,c,d,e,f); - HOST_c2l(data,l); T1 = X[3] = l; ROUND_00_15(3,f,g,h,a,b,c,d,e); - HOST_c2l(data,l); T1 = X[4] = l; ROUND_00_15(4,e,f,g,h,a,b,c,d); - HOST_c2l(data,l); T1 = X[5] = l; ROUND_00_15(5,d,e,f,g,h,a,b,c); - HOST_c2l(data,l); T1 = X[6] = l; ROUND_00_15(6,c,d,e,f,g,h,a,b); - HOST_c2l(data,l); T1 = X[7] = l; ROUND_00_15(7,b,c,d,e,f,g,h,a); - HOST_c2l(data,l); T1 = X[8] = l; ROUND_00_15(8,a,b,c,d,e,f,g,h); - HOST_c2l(data,l); T1 = X[9] = l; ROUND_00_15(9,h,a,b,c,d,e,f,g); - HOST_c2l(data,l); T1 = X[10] = l; ROUND_00_15(10,g,h,a,b,c,d,e,f); - HOST_c2l(data,l); T1 = X[11] = l; ROUND_00_15(11,f,g,h,a,b,c,d,e); - HOST_c2l(data,l); T1 = X[12] = l; ROUND_00_15(12,e,f,g,h,a,b,c,d); - HOST_c2l(data,l); T1 = X[13] = l; ROUND_00_15(13,d,e,f,g,h,a,b,c); - HOST_c2l(data,l); T1 = X[14] = l; ROUND_00_15(14,c,d,e,f,g,h,a,b); - HOST_c2l(data,l); T1 = X[15] = l; ROUND_00_15(15,b,c,d,e,f,g,h,a); - } - - for (i=16;i<64;i+=8) - { - ROUND_16_63(i+0,a,b,c,d,e,f,g,h,X); - ROUND_16_63(i+1,h,a,b,c,d,e,f,g,X); - ROUND_16_63(i+2,g,h,a,b,c,d,e,f,X); - ROUND_16_63(i+3,f,g,h,a,b,c,d,e,X); - ROUND_16_63(i+4,e,f,g,h,a,b,c,d,X); - ROUND_16_63(i+5,d,e,f,g,h,a,b,c,X); - ROUND_16_63(i+6,c,d,e,f,g,h,a,b,X); - ROUND_16_63(i+7,b,c,d,e,f,g,h,a,X); - } - - ctx->h[0] += a; ctx->h[1] += b; ctx->h[2] += c; ctx->h[3] += d; - ctx->h[4] += e; ctx->h[5] += f; ctx->h[6] += g; ctx->h[7] += h; - - } - } - -/* - * Idea is to trade couple of cycles for some space. On IA-32 we save - * about 4K in "big footprint" case. In "small footprint" case any gain - * is appreciated:-) - */ -void HASH_BLOCK_HOST_ORDER (SHA256_CTX *ctx, const void *in, size_t num) -{ sha256_block (ctx,in,num,1); } - -void HASH_BLOCK_DATA_ORDER (SHA256_CTX *ctx, const void *in, size_t num) -{ sha256_block (ctx,in,num,0); } - - diff --git a/src/libutil/sha256.h b/src/libutil/sha256.h deleted file mode 100644 index 0686b84f0e..0000000000 --- a/src/libutil/sha256.h +++ /dev/null @@ -1,35 +0,0 @@ -#ifndef _SHA256_H -#define _SHA256_H 1 - -#include <inttypes.h> - -#define SHA_LBLOCK 16 -#define SHA_CBLOCK (SHA_LBLOCK*4) /* SHA treats input data as a - * contiguous array of 32 bit - * wide big-endian values. */ - -#define SHA256_CBLOCK (SHA_LBLOCK*4) /* SHA-256 treats input data as a - * contiguous array of 32 bit - * wide big-endian values. */ -#define SHA224_DIGEST_LENGTH 28 -#define SHA256_DIGEST_LENGTH 32 - -typedef struct SHA256state_st - { - uint32_t h[8]; - uint32_t Nl,Nh; - uint32_t data[SHA_LBLOCK]; - unsigned int num,md_len; - } SHA256_CTX; - -int SHA224_Init(SHA256_CTX *c); -int SHA224_Update(SHA256_CTX *c, const void *data, size_t len); -int SHA224_Final(unsigned char *md, SHA256_CTX *c); -unsigned char *SHA224(const unsigned char *d, size_t n,unsigned char *md); -int SHA256_Init(SHA256_CTX *c); -int SHA256_Update(SHA256_CTX *c, const void *data, size_t len); -int SHA256_Final(unsigned char *md, SHA256_CTX *c); -unsigned char *SHA256(const unsigned char *d, size_t n,unsigned char *md); -void SHA256_Transform(SHA256_CTX *c, const unsigned char *data); - -#endif diff --git a/src/libutil/types.hh b/src/libutil/types.hh deleted file mode 100644 index 4b5ce9a78c..0000000000 --- a/src/libutil/types.hh +++ /dev/null @@ -1,86 +0,0 @@ -#pragma once - -#include "config.h" - -#include <string> -#include <list> -#include <set> - -#include <boost/format.hpp> - - -namespace nix { - - -/* Inherit some names from other namespaces for convenience. */ -using std::string; -using std::list; -using std::set; -using std::vector; -using boost::format; - - -struct FormatOrString -{ - string s; - FormatOrString(const string & s) : s(s) { }; - FormatOrString(const format & f) : s(f.str()) { }; - FormatOrString(const char * s) : s(s) { }; -}; - - -/* BaseError should generally not be caught, as it has Interrupted as - a subclass. Catch Error instead. */ -class BaseError : public std::exception -{ -protected: - string prefix_; // used for location traces etc. - string err; -public: - unsigned int status; // exit status - BaseError(const FormatOrString & fs, unsigned int status = 1); - ~BaseError() throw () { }; - const char * what() const throw () { return err.c_str(); } - const string & msg() const throw () { return err; } - const string & prefix() const throw () { return prefix_; } - BaseError & addPrefix(const FormatOrString & fs); -}; - -#define MakeError(newClass, superClass) \ - class newClass : public superClass \ - { \ - public: \ - newClass(const FormatOrString & fs, unsigned int status = 1) : superClass(fs, status) { }; \ - }; - -MakeError(Error, BaseError) - -class SysError : public Error -{ -public: - int errNo; - SysError(const FormatOrString & fs); -}; - - -typedef list<string> Strings; -typedef set<string> StringSet; - - -/* Paths are just strings. */ -typedef string Path; -typedef list<Path> Paths; -typedef set<Path> PathSet; - - -typedef enum { - lvlError = 0, - lvlInfo, - lvlTalkative, - lvlChatty, - lvlDebug, - lvlVomit -} Verbosity; - - -} diff --git a/src/libutil/util.cc b/src/libutil/util.cc deleted file mode 100644 index 15c462ce4e..0000000000 --- a/src/libutil/util.cc +++ /dev/null @@ -1,1105 +0,0 @@ -#include "config.h" - -#include <iostream> -#include <cerrno> -#include <cstdio> -#include <cstdlib> -#include <sstream> -#include <cstring> - -#include <sys/wait.h> -#include <unistd.h> -#include <fcntl.h> -#include <limits.h> - -#ifdef __APPLE__ -#include <sys/syscall.h> -#endif - -#include "util.hh" - - -extern char * * environ; - - -namespace nix { - - -BaseError::BaseError(const FormatOrString & fs, unsigned int status) - : status(status) -{ - err = fs.s; -} - - -BaseError & BaseError::addPrefix(const FormatOrString & fs) -{ - prefix_ = fs.s + prefix_; - return *this; -} - - -SysError::SysError(const FormatOrString & fs) - : Error(format("%1%: %2%") % fs.s % strerror(errno)) - , errNo(errno) -{ -} - - -string getEnv(const string & key, const string & def) -{ - char * value = getenv(key.c_str()); - return value ? string(value) : def; -} - - -Path absPath(Path path, Path dir) -{ - if (path[0] != '/') { - if (dir == "") { -#ifdef __GNU__ - /* GNU (aka. GNU/Hurd) doesn't have any limitation on path - lengths and doesn't define `PATH_MAX'. */ - char *buf = getcwd(NULL, 0); - if (buf == NULL) -#else - char buf[PATH_MAX]; - if (!getcwd(buf, sizeof(buf))) -#endif - throw SysError("cannot get cwd"); - dir = buf; -#ifdef __GNU__ - free(buf); -#endif - } - path = dir + "/" + path; - } - return canonPath(path); -} - - -Path canonPath(const Path & path, bool resolveSymlinks) -{ - string s; - - if (path[0] != '/') - throw Error(format("not an absolute path: `%1%'") % path); - - string::const_iterator i = path.begin(), end = path.end(); - string temp; - - /* Count the number of times we follow a symlink and stop at some - arbitrary (but high) limit to prevent infinite loops. */ - unsigned int followCount = 0, maxFollow = 1024; - - while (1) { - - /* Skip slashes. */ - while (i != end && *i == '/') i++; - if (i == end) break; - - /* Ignore `.'. */ - if (*i == '.' && (i + 1 == end || i[1] == '/')) - i++; - - /* If `..', delete the last component. */ - else if (*i == '.' && i + 1 < end && i[1] == '.' && - (i + 2 == end || i[2] == '/')) - { - if (!s.empty()) s.erase(s.rfind('/')); - i += 2; - } - - /* Normal component; copy it. */ - else { - s += '/'; - while (i != end && *i != '/') s += *i++; - - /* If s points to a symlink, resolve it and restart (since - the symlink target might contain new symlinks). */ - if (resolveSymlinks && isLink(s)) { - if (++followCount >= maxFollow) - throw Error(format("infinite symlink recursion in path `%1%'") % path); - temp = absPath(readLink(s), dirOf(s)) - + string(i, end); - i = temp.begin(); /* restart */ - end = temp.end(); - s = ""; - /* !!! potential for infinite loop */ - } - } - } - - return s.empty() ? "/" : s; -} - - -Path dirOf(const Path & path) -{ - Path::size_type pos = path.rfind('/'); - if (pos == string::npos) - throw Error(format("invalid file name `%1%'") % path); - return pos == 0 ? "/" : Path(path, 0, pos); -} - - -string baseNameOf(const Path & path) -{ - Path::size_type pos = path.rfind('/'); - if (pos == string::npos) - throw Error(format("invalid file name `%1%'") % path); - return string(path, pos + 1); -} - - -bool isInDir(const Path & path, const Path & dir) -{ - return path[0] == '/' - && string(path, 0, dir.size()) == dir - && path.size() >= dir.size() + 2 - && path[dir.size()] == '/'; -} - - -struct stat lstat(const Path & path) -{ - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError(format("getting status of `%1%'") % path); - return st; -} - - -bool pathExists(const Path & path) -{ - int res; - struct stat st; - res = lstat(path.c_str(), &st); - if (!res) return true; - if (errno != ENOENT && errno != ENOTDIR) - throw SysError(format("getting status of %1%") % path); - return false; -} - - -Path readLink(const Path & path) -{ - checkInterrupt(); - struct stat st = lstat(path); - if (!S_ISLNK(st.st_mode)) - throw Error(format("`%1%' is not a symlink") % path); - char buf[st.st_size]; - if (readlink(path.c_str(), buf, st.st_size) != st.st_size) - throw SysError(format("reading symbolic link `%1%'") % path); - return string(buf, st.st_size); -} - - -bool isLink(const Path & path) -{ - struct stat st = lstat(path); - return S_ISLNK(st.st_mode); -} - - -Strings readDirectory(const Path & path) -{ - Strings names; - - AutoCloseDir dir = opendir(path.c_str()); - if (!dir) throw SysError(format("opening directory `%1%'") % path); - - struct dirent * dirent; - while (errno = 0, dirent = readdir(dir)) { /* sic */ - checkInterrupt(); - string name = dirent->d_name; - if (name == "." || name == "..") continue; - names.push_back(name); - } - if (errno) throw SysError(format("reading directory `%1%'") % path); - - return names; -} - - -string readFile(int fd) -{ - struct stat st; - if (fstat(fd, &st) == -1) - throw SysError("statting file"); - - unsigned char * buf = new unsigned char[st.st_size]; - AutoDeleteArray<unsigned char> d(buf); - readFull(fd, buf, st.st_size); - - return string((char *) buf, st.st_size); -} - - -string readFile(const Path & path, bool drain) -{ - AutoCloseFD fd = open(path.c_str(), O_RDONLY); - if (fd == -1) - throw SysError(format("opening file `%1%'") % path); - return drain ? drainFD(fd) : readFile(fd); -} - - -void writeFile(const Path & path, const string & s) -{ - AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0666); - if (fd == -1) - throw SysError(format("opening file `%1%'") % path); - writeFull(fd, (unsigned char *) s.data(), s.size()); -} - - -string readLine(int fd) -{ - string s; - while (1) { - checkInterrupt(); - char ch; - ssize_t rd = read(fd, &ch, 1); - if (rd == -1) { - if (errno != EINTR) - throw SysError("reading a line"); - } else if (rd == 0) - throw EndOfFile("unexpected EOF reading a line"); - else { - if (ch == '\n') return s; - s += ch; - } - } -} - - -void writeLine(int fd, string s) -{ - s += '\n'; - writeFull(fd, (const unsigned char *) s.data(), s.size()); -} - - -static void _deletePath(const Path & path, unsigned long long & bytesFreed) -{ - checkInterrupt(); - - printMsg(lvlVomit, format("%1%") % path); - - struct stat st = lstat(path); - - if (!S_ISDIR(st.st_mode) && st.st_nlink == 1) - bytesFreed += st.st_blocks * 512; - - if (S_ISDIR(st.st_mode)) { - Strings names = readDirectory(path); - - /* Make the directory writable. */ - if (!(st.st_mode & S_IWUSR)) { - if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1) - throw SysError(format("making `%1%' writable") % path); - } - - for (Strings::iterator i = names.begin(); i != names.end(); ++i) - _deletePath(path + "/" + *i, bytesFreed); - } - - if (remove(path.c_str()) == -1) - throw SysError(format("cannot unlink `%1%'") % path); -} - - -void deletePath(const Path & path) -{ - unsigned long long dummy; - deletePath(path, dummy); -} - - -void deletePath(const Path & path, unsigned long long & bytesFreed) -{ - startNest(nest, lvlDebug, - format("recursively deleting path `%1%'") % path); - bytesFreed = 0; - _deletePath(path, bytesFreed); -} - - -static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, - int & counter) -{ - tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR", "/tmp") : tmpRoot, true); - if (includePid) - return (format("%1%/%2%-%3%-%4%") % tmpRoot % prefix % getpid() % counter++).str(); - else - return (format("%1%/%2%-%3%") % tmpRoot % prefix % counter++).str(); -} - - -Path createTempDir(const Path & tmpRoot, const Path & prefix, - bool includePid, bool useGlobalCounter, mode_t mode) -{ - static int globalCounter = 0; - int localCounter = 0; - int & counter(useGlobalCounter ? globalCounter : localCounter); - - while (1) { - checkInterrupt(); - Path tmpDir = tempName(tmpRoot, prefix, includePid, counter); - if (mkdir(tmpDir.c_str(), mode) == 0) { - /* Explicitly set the group of the directory. This is to - work around around problems caused by BSD's group - ownership semantics (directories inherit the group of - the parent). For instance, the group of /tmp on - FreeBSD is "wheel", so all directories created in /tmp - will be owned by "wheel"; but if the user is not in - "wheel", then "tar" will fail to unpack archives that - have the setgid bit set on directories. */ - if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0) - throw SysError(format("setting group of directory `%1%'") % tmpDir); - return tmpDir; - } - if (errno != EEXIST) - throw SysError(format("creating directory `%1%'") % tmpDir); - } -} - - -Paths createDirs(const Path & path) -{ - Paths created; - if (path == "/") return created; - - struct stat st; - if (lstat(path.c_str(), &st) == -1) { - created = createDirs(dirOf(path)); - if (mkdir(path.c_str(), 0777) == -1 && errno != EEXIST) - throw SysError(format("creating directory `%1%'") % path); - st = lstat(path); - created.push_back(path); - } - - if (!S_ISDIR(st.st_mode)) throw Error(format("`%1%' is not a directory") % path); - - return created; -} - - -void createSymlink(const Path & target, const Path & link) -{ - if (symlink(target.c_str(), link.c_str())) - throw SysError(format("creating symlink from `%1%' to `%2%'") % link % target); -} - - -LogType logType = ltPretty; -Verbosity verbosity = lvlInfo; - -static int nestingLevel = 0; - - -Nest::Nest() -{ - nest = false; -} - - -Nest::~Nest() -{ - close(); -} - - -static string escVerbosity(Verbosity level) -{ - return int2String((int) level); -} - - -void Nest::open(Verbosity level, const FormatOrString & fs) -{ - if (level <= verbosity) { - if (logType == ltEscapes) - std::cerr << "\033[" << escVerbosity(level) << "p" - << fs.s << "\n"; - else - printMsg_(level, fs); - nest = true; - nestingLevel++; - } -} - - -void Nest::close() -{ - if (nest) { - nestingLevel--; - if (logType == ltEscapes) - std::cerr << "\033[q"; - nest = false; - } -} - - -void printMsg_(Verbosity level, const FormatOrString & fs) -{ - checkInterrupt(); - if (level > verbosity) return; - string prefix; - if (logType == ltPretty) - for (int i = 0; i < nestingLevel; i++) - prefix += "| "; - else if (logType == ltEscapes && level != lvlInfo) - prefix = "\033[" + escVerbosity(level) + "s"; - string s = (format("%1%%2%\n") % prefix % fs.s).str(); - writeToStderr(s); -} - - -void warnOnce(bool & haveWarned, const FormatOrString & fs) -{ - if (!haveWarned) { - printMsg(lvlError, format("warning: %1%") % fs.s); - haveWarned = true; - } -} - - -void writeToStderr(const string & s) -{ - try { - _writeToStderr((const unsigned char *) s.data(), s.size()); - } catch (SysError & e) { - /* Ignore failing writes to stderr if we're in an exception - handler, otherwise throw an exception. We need to ignore - write errors in exception handlers to ensure that cleanup - code runs to completion if the other side of stderr has - been closed unexpectedly. */ - if (!std::uncaught_exception()) throw; - } -} - - -static void defaultWriteToStderr(const unsigned char * buf, size_t count) -{ - writeFull(STDERR_FILENO, buf, count); -} - - -void (*_writeToStderr) (const unsigned char * buf, size_t count) = defaultWriteToStderr; - - -void readFull(int fd, unsigned char * buf, size_t count) -{ - while (count) { - checkInterrupt(); - ssize_t res = read(fd, (char *) buf, count); - if (res == -1) { - if (errno == EINTR) continue; - throw SysError("reading from file"); - } - if (res == 0) throw EndOfFile("unexpected end-of-file"); - count -= res; - buf += res; - } -} - - -void writeFull(int fd, const unsigned char * buf, size_t count) -{ - while (count) { - checkInterrupt(); - ssize_t res = write(fd, (char *) buf, count); - if (res == -1) { - if (errno == EINTR) continue; - throw SysError("writing to file"); - } - count -= res; - buf += res; - } -} - - -string drainFD(int fd) -{ - string result; - unsigned char buffer[4096]; - while (1) { - checkInterrupt(); - ssize_t rd = read(fd, buffer, sizeof buffer); - if (rd == -1) { - if (errno != EINTR) - throw SysError("reading from file"); - } - else if (rd == 0) break; - else result.append((char *) buffer, rd); - } - return result; -} - - - -////////////////////////////////////////////////////////////////////// - - -AutoDelete::AutoDelete(const string & p, bool recursive) : path(p) -{ - del = true; - this->recursive = recursive; -} - -AutoDelete::~AutoDelete() -{ - try { - if (del) { - if (recursive) - deletePath(path); - else { - if (remove(path.c_str()) == -1) - throw SysError(format("cannot unlink `%1%'") % path); - } - } - } catch (...) { - ignoreException(); - } -} - -void AutoDelete::cancel() -{ - del = false; -} - - - -////////////////////////////////////////////////////////////////////// - - -AutoCloseFD::AutoCloseFD() -{ - fd = -1; -} - - -AutoCloseFD::AutoCloseFD(int fd) -{ - this->fd = fd; -} - - -AutoCloseFD::AutoCloseFD(const AutoCloseFD & fd) -{ - /* Copying an AutoCloseFD isn't allowed (who should get to close - it?). But as an edge case, allow copying of closed - AutoCloseFDs. This is necessary due to tiresome reasons - involving copy constructor use on default object values in STL - containers (like when you do `map[value]' where value isn't in - the map yet). */ - this->fd = fd.fd; - if (this->fd != -1) abort(); -} - - -AutoCloseFD::~AutoCloseFD() -{ - try { - close(); - } catch (...) { - ignoreException(); - } -} - - -void AutoCloseFD::operator =(int fd) -{ - if (this->fd != fd) close(); - this->fd = fd; -} - - -AutoCloseFD::operator int() const -{ - return fd; -} - - -void AutoCloseFD::close() -{ - if (fd != -1) { - if (::close(fd) == -1) - /* This should never happen. */ - throw SysError(format("closing file descriptor %1%") % fd); - fd = -1; - } -} - - -bool AutoCloseFD::isOpen() -{ - return fd != -1; -} - - -/* Pass responsibility for closing this fd to the caller. */ -int AutoCloseFD::borrow() -{ - int oldFD = fd; - fd = -1; - return oldFD; -} - - -void Pipe::create() -{ - int fds[2]; - if (pipe(fds) != 0) throw SysError("creating pipe"); - readSide = fds[0]; - writeSide = fds[1]; - closeOnExec(readSide); - closeOnExec(writeSide); -} - - - -////////////////////////////////////////////////////////////////////// - - -AutoCloseDir::AutoCloseDir() -{ - dir = 0; -} - - -AutoCloseDir::AutoCloseDir(DIR * dir) -{ - this->dir = dir; -} - - -AutoCloseDir::~AutoCloseDir() -{ - close(); -} - - -void AutoCloseDir::operator =(DIR * dir) -{ - this->dir = dir; -} - - -AutoCloseDir::operator DIR *() -{ - return dir; -} - - -void AutoCloseDir::close() -{ - if (dir) { - closedir(dir); - dir = 0; - } -} - - -////////////////////////////////////////////////////////////////////// - - -Pid::Pid() -{ - pid = -1; - separatePG = false; - killSignal = SIGKILL; -} - - -Pid::~Pid() -{ - kill(); -} - - -void Pid::operator =(pid_t pid) -{ - if (this->pid != pid) kill(); - this->pid = pid; - killSignal = SIGKILL; // reset signal to default -} - - -Pid::operator pid_t() -{ - return pid; -} - - -void Pid::kill() -{ - if (pid == -1 || pid == 0) return; - - printMsg(lvlError, format("killing process %1%") % pid); - - /* Send the requested signal to the child. If it has its own - process group, send the signal to every process in the child - process group (which hopefully includes *all* its children). */ - if (::kill(separatePG ? -pid : pid, killSignal) != 0) - printMsg(lvlError, (SysError(format("killing process %1%") % pid).msg())); - - /* Wait until the child dies, disregarding the exit status. */ - int status; - while (waitpid(pid, &status, 0) == -1) { - checkInterrupt(); - if (errno != EINTR) { - printMsg(lvlError, - (SysError(format("waiting for process %1%") % pid).msg())); - break; - } - } - - pid = -1; -} - - -int Pid::wait(bool block) -{ - assert(pid != -1); - while (1) { - int status; - int res = waitpid(pid, &status, block ? 0 : WNOHANG); - if (res == pid) { - pid = -1; - return status; - } - if (res == 0 && !block) return -1; - if (errno != EINTR) - throw SysError("cannot get child exit status"); - checkInterrupt(); - } -} - - -void Pid::setSeparatePG(bool separatePG) -{ - this->separatePG = separatePG; -} - - -void Pid::setKillSignal(int signal) -{ - this->killSignal = signal; -} - - -void killUser(uid_t uid) -{ - debug(format("killing all processes running under uid `%1%'") % uid); - - assert(uid != 0); /* just to be safe... */ - - /* The system call kill(-1, sig) sends the signal `sig' to all - users to which the current process can send signals. So we - fork a process, switch to uid, and send a mass kill. */ - - Pid pid; - pid = fork(); - switch (pid) { - - case -1: - throw SysError("unable to fork"); - - case 0: - try { /* child */ - - if (setuid(uid) == -1) - throw SysError("setting uid"); - - while (true) { -#ifdef __APPLE__ - /* OSX's kill syscall takes a third parameter that, among other - things, determines if kill(-1, signo) affects the calling - process. In the OSX libc, it's set to true, which means - "follow POSIX", which we don't want here - */ - if (syscall(SYS_kill, -1, SIGKILL, false) == 0) break; -#else - if (kill(-1, SIGKILL) == 0) break; -#endif - if (errno == ESRCH) break; /* no more processes */ - if (errno != EINTR) - throw SysError(format("cannot kill processes for uid `%1%'") % uid); - } - - } catch (std::exception & e) { - writeToStderr((format("killing processes belonging to uid `%1%': %2%\n") % uid % e.what()).str()); - _exit(1); - } - _exit(0); - } - - /* parent */ - int status = pid.wait(true); - if (status != 0) - throw Error(format("cannot kill processes for uid `%1%': %2%") % uid % statusToString(status)); - - /* !!! We should really do some check to make sure that there are - no processes left running under `uid', but there is no portable - way to do so (I think). The most reliable way may be `ps -eo - uid | grep -q $uid'. */ -} - - -////////////////////////////////////////////////////////////////////// - - -string runProgram(Path program, bool searchPath, const Strings & args) -{ - checkInterrupt(); - - std::vector<const char *> cargs; /* careful with c_str()! */ - cargs.push_back(program.c_str()); - for (Strings::const_iterator i = args.begin(); i != args.end(); ++i) - cargs.push_back(i->c_str()); - cargs.push_back(0); - - /* Create a pipe. */ - Pipe pipe; - pipe.create(); - - /* Fork. */ - Pid pid; - pid = maybeVfork(); - - switch (pid) { - - case -1: - throw SysError("unable to fork"); - - case 0: /* child */ - try { - if (dup2(pipe.writeSide, STDOUT_FILENO) == -1) - throw SysError("dupping stdout"); - - if (searchPath) - execvp(program.c_str(), (char * *) &cargs[0]); - else - execv(program.c_str(), (char * *) &cargs[0]); - throw SysError(format("executing `%1%'") % program); - - } catch (std::exception & e) { - writeToStderr("error: " + string(e.what()) + "\n"); - } - _exit(1); - } - - /* Parent. */ - - pipe.writeSide.close(); - - string result = drainFD(pipe.readSide); - - /* Wait for the child to finish. */ - int status = pid.wait(true); - if (!statusOk(status)) - throw Error(format("program `%1%' %2%") - % program % statusToString(status)); - - return result; -} - - -void closeMostFDs(const set<int> & exceptions) -{ - int maxFD = 0; - maxFD = sysconf(_SC_OPEN_MAX); - for (int fd = 0; fd < maxFD; ++fd) - if (fd != STDIN_FILENO && fd != STDOUT_FILENO && fd != STDERR_FILENO - && exceptions.find(fd) == exceptions.end()) - close(fd); /* ignore result */ -} - - -void closeOnExec(int fd) -{ - int prev; - if ((prev = fcntl(fd, F_GETFD, 0)) == -1 || - fcntl(fd, F_SETFD, prev | FD_CLOEXEC) == -1) - throw SysError("setting close-on-exec flag"); -} - - -#if HAVE_VFORK -pid_t (*maybeVfork)() = vfork; -#else -pid_t (*maybeVfork)() = fork; -#endif - - -////////////////////////////////////////////////////////////////////// - - -volatile sig_atomic_t _isInterrupted = 0; - -void _interrupted() -{ - /* Block user interrupts while an exception is being handled. - Throwing an exception while another exception is being handled - kills the program! */ - if (!std::uncaught_exception()) { - _isInterrupted = 0; - throw Interrupted("interrupted by the user"); - } -} - - - -////////////////////////////////////////////////////////////////////// - - -template<class C> C tokenizeString(const string & s, const string & separators) -{ - C result; - string::size_type pos = s.find_first_not_of(separators, 0); - while (pos != string::npos) { - string::size_type end = s.find_first_of(separators, pos + 1); - if (end == string::npos) end = s.size(); - string token(s, pos, end - pos); - result.insert(result.end(), token); - pos = s.find_first_not_of(separators, end); - } - return result; -} - -template Strings tokenizeString(const string & s, const string & separators); -template StringSet tokenizeString(const string & s, const string & separators); -template vector<string> tokenizeString(const string & s, const string & separators); - - -string concatStringsSep(const string & sep, const Strings & ss) -{ - string s; - foreach (Strings::const_iterator, i, ss) { - if (s.size() != 0) s += sep; - s += *i; - } - return s; -} - - -string concatStringsSep(const string & sep, const StringSet & ss) -{ - string s; - foreach (StringSet::const_iterator, i, ss) { - if (s.size() != 0) s += sep; - s += *i; - } - return s; -} - - -string chomp(const string & s) -{ - size_t i = s.find_last_not_of(" \n\r\t"); - return i == string::npos ? "" : string(s, 0, i + 1); -} - - -string statusToString(int status) -{ - if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { - if (WIFEXITED(status)) - return (format("failed with exit code %1%") % WEXITSTATUS(status)).str(); - else if (WIFSIGNALED(status)) { - int sig = WTERMSIG(status); -#if HAVE_STRSIGNAL - const char * description = strsignal(sig); - return (format("failed due to signal %1% (%2%)") % sig % description).str(); -#else - return (format("failed due to signal %1%") % sig).str(); -#endif - } - else - return "died abnormally"; - } else return "succeeded"; -} - - -bool statusOk(int status) -{ - return WIFEXITED(status) && WEXITSTATUS(status) == 0; -} - - -bool hasSuffix(const string & s, const string & suffix) -{ - return s.size() >= suffix.size() && string(s, s.size() - suffix.size()) == suffix; -} - - -void expect(std::istream & str, const string & s) -{ - char s2[s.size()]; - str.read(s2, s.size()); - if (string(s2, s.size()) != s) - throw Error(format("expected string `%1%'") % s); -} - - -string parseString(std::istream & str) -{ - string res; - expect(str, "\""); - int c; - while ((c = str.get()) != '"') - if (c == '\\') { - c = str.get(); - if (c == 'n') res += '\n'; - else if (c == 'r') res += '\r'; - else if (c == 't') res += '\t'; - else res += c; - } - else res += c; - return res; -} - - -bool endOfList(std::istream & str) -{ - if (str.peek() == ',') { - str.get(); - return false; - } - if (str.peek() == ']') { - str.get(); - return true; - } - return false; -} - - -string decodeOctalEscaped(const string & s) -{ - string r; - for (string::const_iterator i = s.begin(); i != s.end(); ) { - if (*i != '\\') { r += *i++; continue; } - unsigned char c = 0; - ++i; - while (i != s.end() && *i >= '0' && *i < '8') - c = c * 8 + (*i++ - '0'); - r += c; - } - return r; -} - - -void ignoreException() -{ - try { - throw; - } catch (std::exception & e) { - printMsg(lvlError, format("error (ignored): %1%") % e.what()); - } -} - - -} diff --git a/src/libutil/util.hh b/src/libutil/util.hh deleted file mode 100644 index 8bedfea9a0..0000000000 --- a/src/libutil/util.hh +++ /dev/null @@ -1,349 +0,0 @@ -#pragma once - -#include "types.hh" - -#include <sys/types.h> -#include <sys/stat.h> -#include <dirent.h> -#include <unistd.h> -#include <signal.h> - -#include <cstdio> - - -namespace nix { - - -#define foreach(it_type, it, collection) \ - for (it_type it = (collection).begin(); it != (collection).end(); ++it) - -#define foreach_reverse(it_type, it, collection) \ - for (it_type it = (collection).rbegin(); it != (collection).rend(); ++it) - - -/* Return an environment variable. */ -string getEnv(const string & key, const string & def = ""); - -/* Return an absolutized path, resolving paths relative to the - specified directory, or the current directory otherwise. The path - is also canonicalised. */ -Path absPath(Path path, Path dir = ""); - -/* Canonicalise a path by removing all `.' or `..' components and - double or trailing slashes. Optionally resolves all symlink - components such that each component of the resulting path is *not* - a symbolic link. */ -Path canonPath(const Path & path, bool resolveSymlinks = false); - -/* Return the directory part of the given canonical path, i.e., - everything before the final `/'. If the path is the root or an - immediate child thereof (e.g., `/foo'), this means an empty string - is returned. */ -Path dirOf(const Path & path); - -/* Return the base name of the given canonical path, i.e., everything - following the final `/'. */ -string baseNameOf(const Path & path); - -/* Check whether a given path is a descendant of the given - directory. */ -bool isInDir(const Path & path, const Path & dir); - -/* Get status of `path'. */ -struct stat lstat(const Path & path); - -/* Return true iff the given path exists. */ -bool pathExists(const Path & path); - -/* Read the contents (target) of a symbolic link. The result is not - in any way canonicalised. */ -Path readLink(const Path & path); - -bool isLink(const Path & path); - -/* Read the contents of a directory. The entries `.' and `..' are - removed. */ -Strings readDirectory(const Path & path); - -/* Read the contents of a file into a string. */ -string readFile(int fd); -string readFile(const Path & path, bool drain = false); - -/* Write a string to a file. */ -void writeFile(const Path & path, const string & s); - -/* Read a line from a file descriptor. */ -string readLine(int fd); - -/* Write a line to a file descriptor. */ -void writeLine(int fd, string s); - -/* Delete a path; i.e., in the case of a directory, it is deleted - recursively. Don't use this at home, kids. The second variant - returns the number of bytes and blocks freed. */ -void deletePath(const Path & path); - -void deletePath(const Path & path, unsigned long long & bytesFreed); - -/* Create a temporary directory. */ -Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", - bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); - -/* Create a directory and all its parents, if necessary. Returns the - list of created directories, in order of creation. */ -Paths createDirs(const Path & path); - -/* Create a symlink. */ -void createSymlink(const Path & target, const Path & link); - - -template<class T, class A> -T singleton(const A & a) -{ - T t; - t.insert(a); - return t; -} - - -/* Messages. */ - - -typedef enum { - ltPretty, /* nice, nested output */ - ltEscapes, /* nesting indicated using escape codes (for log2xml) */ - ltFlat /* no nesting */ -} LogType; - -extern LogType logType; -extern Verbosity verbosity; /* suppress msgs > this */ - -class Nest -{ -private: - bool nest; -public: - Nest(); - ~Nest(); - void open(Verbosity level, const FormatOrString & fs); - void close(); -}; - -void printMsg_(Verbosity level, const FormatOrString & fs); - -#define startNest(varName, level, f) \ - Nest varName; \ - if (level <= verbosity) { \ - varName.open(level, (f)); \ - } - -#define printMsg(level, f) \ - do { \ - if (level <= verbosity) { \ - printMsg_(level, (f)); \ - } \ - } while (0) - -#define debug(f) printMsg(lvlDebug, f) - -void warnOnce(bool & haveWarned, const FormatOrString & fs); - -void writeToStderr(const string & s); - -extern void (*_writeToStderr) (const unsigned char * buf, size_t count); - - -/* Wrappers arount read()/write() that read/write exactly the - requested number of bytes. */ -void readFull(int fd, unsigned char * buf, size_t count); -void writeFull(int fd, const unsigned char * buf, size_t count); - -MakeError(EndOfFile, Error) - - -/* Read a file descriptor until EOF occurs. */ -string drainFD(int fd); - - - -/* Automatic cleanup of resources. */ - - -template <class T> -struct AutoDeleteArray -{ - T * p; - AutoDeleteArray(T * p) : p(p) { } - ~AutoDeleteArray() - { - delete [] p; - } -}; - - -class AutoDelete -{ - Path path; - bool del; - bool recursive; -public: - AutoDelete(const Path & p, bool recursive = true); - ~AutoDelete(); - void cancel(); -}; - - -class AutoCloseFD -{ - int fd; -public: - AutoCloseFD(); - AutoCloseFD(int fd); - AutoCloseFD(const AutoCloseFD & fd); - ~AutoCloseFD(); - void operator =(int fd); - operator int() const; - void close(); - bool isOpen(); - int borrow(); -}; - - -class Pipe -{ -public: - AutoCloseFD readSide, writeSide; - void create(); -}; - - -class AutoCloseDir -{ - DIR * dir; -public: - AutoCloseDir(); - AutoCloseDir(DIR * dir); - ~AutoCloseDir(); - void operator =(DIR * dir); - operator DIR *(); - void close(); -}; - - -class Pid -{ - pid_t pid; - bool separatePG; - int killSignal; -public: - Pid(); - ~Pid(); - void operator =(pid_t pid); - operator pid_t(); - void kill(); - int wait(bool block); - void setSeparatePG(bool separatePG); - void setKillSignal(int signal); -}; - - -/* Kill all processes running under the specified uid by sending them - a SIGKILL. */ -void killUser(uid_t uid); - - -/* Run a program and return its stdout in a string (i.e., like the - shell backtick operator). */ -string runProgram(Path program, bool searchPath = false, - const Strings & args = Strings()); - -/* Close all file descriptors except stdin, stdout, stderr, and those - listed in the given set. Good practice in child processes. */ -void closeMostFDs(const set<int> & exceptions); - -/* Set the close-on-exec flag for the given file descriptor. */ -void closeOnExec(int fd); - -/* Call vfork() if available, otherwise fork(). */ -extern pid_t (*maybeVfork)(); - - -/* User interruption. */ - -extern volatile sig_atomic_t _isInterrupted; - -void _interrupted(); - -void inline checkInterrupt() -{ - if (_isInterrupted) _interrupted(); -} - -MakeError(Interrupted, BaseError) - - -/* String tokenizer. */ -template<class C> C tokenizeString(const string & s, const string & separators = " \t\n\r"); - - -/* Concatenate the given strings with a separator between the - elements. */ -string concatStringsSep(const string & sep, const Strings & ss); -string concatStringsSep(const string & sep, const StringSet & ss); - - -/* Remove trailing whitespace from a string. */ -string chomp(const string & s); - - -/* Convert the exit status of a child as returned by wait() into an - error string. */ -string statusToString(int status); - -bool statusOk(int status); - - -/* Parse a string into an integer. */ -template<class N> bool string2Int(const string & s, N & n) -{ - std::istringstream str(s); - str >> n; - return str && str.get() == EOF; -} - -template<class N> string int2String(N n) -{ - std::ostringstream str; - str << n; - return str.str(); -} - - -/* Return true iff `s' ends in `suffix'. */ -bool hasSuffix(const string & s, const string & suffix); - - -/* Read string `s' from stream `str'. */ -void expect(std::istream & str, const string & s); - - -/* Read a C-style string from stream `str'. */ -string parseString(std::istream & str); - - -/* Utility function used to parse legacy ATerms. */ -bool endOfList(std::istream & str); - - -/* Escape a string that contains octal-encoded escape codes such as - used in /etc/fstab and /proc/mounts (e.g. "foo\040bar" decodes to - "foo bar"). */ -string decodeOctalEscaped(const string & s); - - -/* Exception handling in destructors: print an error message, then - ignore the exception. */ -void ignoreException(); - - -} diff --git a/src/libutil/xml-writer.cc b/src/libutil/xml-writer.cc deleted file mode 100644 index 01794001b2..0000000000 --- a/src/libutil/xml-writer.cc +++ /dev/null @@ -1,94 +0,0 @@ -#include <assert.h> - -#include "xml-writer.hh" - - -namespace nix { - - -XMLWriter::XMLWriter(bool indent, std::ostream & output) - : output(output), indent(indent) -{ - output << "<?xml version='1.0' encoding='utf-8'?>" << std::endl; - closed = false; -} - - -XMLWriter::~XMLWriter() -{ - close(); -} - - -void XMLWriter::close() -{ - if (closed) return; - while (!pendingElems.empty()) closeElement(); - closed = true; -} - - -void XMLWriter::indent_(unsigned int depth) -{ - if (!indent) return; - output << string(depth * 2, ' '); -} - - -void XMLWriter::openElement(const string & name, - const XMLAttrs & attrs) -{ - assert(!closed); - indent_(pendingElems.size()); - output << "<" << name; - writeAttrs(attrs); - output << ">"; - if (indent) output << std::endl; - pendingElems.push_back(name); -} - - -void XMLWriter::closeElement() -{ - assert(!pendingElems.empty()); - indent_(pendingElems.size() - 1); - output << "</" << pendingElems.back() << ">"; - if (indent) output << std::endl; - pendingElems.pop_back(); - if (pendingElems.empty()) closed = true; -} - - -void XMLWriter::writeEmptyElement(const string & name, - const XMLAttrs & attrs) -{ - assert(!closed); - indent_(pendingElems.size()); - output << "<" << name; - writeAttrs(attrs); - output << " />"; - if (indent) output << std::endl; -} - - -void XMLWriter::writeAttrs(const XMLAttrs & attrs) -{ - for (XMLAttrs::const_iterator i = attrs.begin(); i != attrs.end(); ++i) { - output << " " << i->first << "=\""; - for (unsigned int j = 0; j < i->second.size(); ++j) { - char c = i->second[j]; - if (c == '"') output << """; - else if (c == '<') output << "<"; - else if (c == '>') output << ">"; - else if (c == '&') output << "&"; - /* Escape newlines to prevent attribute normalisation (see - XML spec, section 3.3.3. */ - else if (c == '\n') output << "
"; - else output << c; - } - output << "\""; - } -} - - -} diff --git a/src/libutil/xml-writer.hh b/src/libutil/xml-writer.hh deleted file mode 100644 index 3cefe3712c..0000000000 --- a/src/libutil/xml-writer.hh +++ /dev/null @@ -1,69 +0,0 @@ -#pragma once - -#include <iostream> -#include <string> -#include <list> -#include <map> - - -namespace nix { - -using std::string; -using std::map; -using std::list; - - -typedef map<string, string> XMLAttrs; - - -class XMLWriter -{ -private: - - std::ostream & output; - - bool indent; - bool closed; - - list<string> pendingElems; - -public: - - XMLWriter(bool indent, std::ostream & output); - ~XMLWriter(); - - void close(); - - void openElement(const string & name, - const XMLAttrs & attrs = XMLAttrs()); - void closeElement(); - - void writeEmptyElement(const string & name, - const XMLAttrs & attrs = XMLAttrs()); - -private: - void writeAttrs(const XMLAttrs & attrs); - - void indent_(unsigned int depth); -}; - - -class XMLOpenElement -{ -private: - XMLWriter & writer; -public: - XMLOpenElement(XMLWriter & writer, const string & name, - const XMLAttrs & attrs = XMLAttrs()) - : writer(writer) - { - writer.openElement(name, attrs); - } - ~XMLOpenElement() - { - writer.closeElement(); - } -}; - - -} diff --git a/src/nix-daemon/local.mk b/src/nix-daemon/local.mk deleted file mode 100644 index db071a3f7c..0000000000 --- a/src/nix-daemon/local.mk +++ /dev/null @@ -1,9 +0,0 @@ -programs += nix-daemon - -nix-daemon_DIR := $(d) - -nix-daemon_SOURCES := $(d)/nix-daemon.cc - -nix-daemon_LIBS = libmain libstore libutil libformat - -$(eval $(call install-symlink, nix-daemon, $(bindir)/nix-worker)) diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc deleted file mode 100644 index 8814fe3155..0000000000 --- a/src/nix-daemon/nix-daemon.cc +++ /dev/null @@ -1,939 +0,0 @@ -#include "shared.hh" -#include "local-store.hh" -#include "util.hh" -#include "serialise.hh" -#include "worker-protocol.hh" -#include "archive.hh" -#include "affinity.hh" -#include "globals.hh" - -#include <cstring> -#include <unistd.h> -#include <signal.h> -#include <sys/types.h> -#include <sys/wait.h> -#include <sys/stat.h> -#include <sys/socket.h> -#include <sys/un.h> -#include <fcntl.h> -#include <errno.h> - -using namespace nix; - - -/* On platforms that have O_ASYNC, we can detect when a client - disconnects and immediately kill any ongoing builds. On platforms - that lack it, we only notice the disconnection the next time we try - to write to the client. So if you have a builder that never - generates output on stdout/stderr, the daemon will never notice - that the client has disconnected until the builder terminates. - - GNU/Hurd does have O_ASYNC, but its Unix-domain socket translator - (pflocal) does not implement F_SETOWN. See - <http://lists.gnu.org/archive/html/bug-guix/2013-07/msg00021.html> for - details.*/ -#if defined O_ASYNC && !defined __GNU__ -#define HAVE_HUP_NOTIFICATION -#ifndef SIGPOLL -#define SIGPOLL SIGIO -#endif -#endif - - -static FdSource from(STDIN_FILENO); -static FdSink to(STDOUT_FILENO); - -bool canSendStderr; -pid_t myPid; - - - -/* This function is called anytime we want to write something to - stderr. If we're in a state where the protocol allows it (i.e., - when canSendStderr), send the message to the client over the - socket. */ -static void tunnelStderr(const unsigned char * buf, size_t count) -{ - /* Don't send the message to the client if we're a child of the - process handling the connection. Otherwise we could screw up - the protocol. It's up to the parent to redirect stderr and - send it to the client somehow (e.g., as in build.cc). */ - if (canSendStderr && myPid == getpid()) { - try { - writeInt(STDERR_NEXT, to); - writeString(buf, count, to); - to.flush(); - } catch (...) { - /* Write failed; that means that the other side is - gone. */ - canSendStderr = false; - throw; - } - } else - writeFull(STDERR_FILENO, buf, count); -} - - -/* Return true if the remote side has closed its end of the - connection, false otherwise. Should not be called on any socket on - which we expect input! */ -static bool isFarSideClosed(int socket) -{ - struct timeval timeout; - timeout.tv_sec = timeout.tv_usec = 0; - - fd_set fds; - FD_ZERO(&fds); - FD_SET(socket, &fds); - - while (select(socket + 1, &fds, 0, 0, &timeout) == -1) - if (errno != EINTR) throw SysError("select()"); - - if (!FD_ISSET(socket, &fds)) return false; - - /* Destructive read to determine whether the select() marked the - socket as readable because there is actual input or because - we've reached EOF (i.e., a read of size 0 is available). */ - char c; - int rd; - if ((rd = read(socket, &c, 1)) > 0) - throw Error("EOF expected (protocol error?)"); - else if (rd == -1 && errno != ECONNRESET) - throw SysError("expected connection reset or EOF"); - - return true; -} - - -/* A SIGPOLL signal is received when data is available on the client - communication socket, or when the client has closed its side of the - socket. This handler is enabled at precisely those moments in the - protocol when we're doing work and the client is supposed to be - quiet. Thus, if we get a SIGPOLL signal, it means that the client - has quit. So we should quit as well. - - Too bad most operating systems don't support the POLL_HUP value for - si_code in siginfo_t. That would make most of the SIGPOLL - complexity unnecessary, i.e., we could just enable SIGPOLL all the - time and wouldn't have to worry about races. */ -static void sigPollHandler(int sigNo) -{ - using namespace std; - try { - /* Check that the far side actually closed. We're still - getting spurious signals every once in a while. I.e., - there is no input available, but we get a signal with - POLL_IN set. Maybe it's delayed or something. */ - if (isFarSideClosed(from.fd)) { - if (!blockInt) { - _isInterrupted = 1; - blockInt = 1; - canSendStderr = false; - const char * s = "SIGPOLL\n"; - write(STDERR_FILENO, s, strlen(s)); - } - } else { - const char * s = "spurious SIGPOLL\n"; - write(STDERR_FILENO, s, strlen(s)); - } - } - catch (Error & e) { - /* Shouldn't happen. */ - string s = "impossible: " + e.msg() + '\n'; - write(STDERR_FILENO, s.data(), s.size()); - throw; - } -} - - -static void setSigPollAction(bool enable) -{ -#ifdef HAVE_HUP_NOTIFICATION - struct sigaction act, oact; - act.sa_handler = enable ? sigPollHandler : SIG_IGN; - sigfillset(&act.sa_mask); - act.sa_flags = 0; - if (sigaction(SIGPOLL, &act, &oact)) - throw SysError("setting handler for SIGPOLL"); -#endif -} - - -/* startWork() means that we're starting an operation for which we - want to send out stderr to the client. */ -static void startWork() -{ - canSendStderr = true; - - /* Handle client death asynchronously. */ - setSigPollAction(true); - - /* Of course, there is a race condition here: the socket could - have closed between when we last read from / wrote to it, and - between the time we set the handler for SIGPOLL. In that case - we won't get the signal. So do a non-blocking select() to find - out if any input is available on the socket. If there is, it - has to be the 0-byte read that indicates that the socket has - closed. */ - if (isFarSideClosed(from.fd)) { - _isInterrupted = 1; - checkInterrupt(); - } -} - - -/* stopWork() means that we're done; stop sending stderr to the - client. */ -static void stopWork(bool success = true, const string & msg = "", unsigned int status = 0) -{ - /* Stop handling async client death; we're going to a state where - we're either sending or receiving from the client, so we'll be - notified of client death anyway. */ - setSigPollAction(false); - - canSendStderr = false; - - if (success) - writeInt(STDERR_LAST, to); - else { - writeInt(STDERR_ERROR, to); - writeString(msg, to); - if (status != 0) writeInt(status, to); - } -} - - -struct TunnelSink : Sink -{ - Sink & to; - TunnelSink(Sink & to) : to(to) { } - virtual void operator () (const unsigned char * data, size_t len) - { - writeInt(STDERR_WRITE, to); - writeString(data, len, to); - } -}; - - -struct TunnelSource : BufferedSource -{ - Source & from; - TunnelSource(Source & from) : from(from) { } - size_t readUnbuffered(unsigned char * data, size_t len) - { - /* Careful: we're going to receive data from the client now, - so we have to disable the SIGPOLL handler. */ - setSigPollAction(false); - canSendStderr = false; - - writeInt(STDERR_READ, to); - writeInt(len, to); - to.flush(); - size_t n = readString(data, len, from); - - startWork(); - if (n == 0) throw EndOfFile("unexpected end-of-file"); - return n; - } -}; - - -/* If the NAR archive contains a single file at top-level, then save - the contents of the file to `s'. Otherwise barf. */ -struct RetrieveRegularNARSink : ParseSink -{ - bool regular; - string s; - - RetrieveRegularNARSink() : regular(true) { } - - void createDirectory(const Path & path) - { - regular = false; - } - - void receiveContents(unsigned char * data, unsigned int len) - { - s.append((const char *) data, len); - } - - void createSymlink(const Path & path, const string & target) - { - regular = false; - } -}; - - -/* Adapter class of a Source that saves all data read to `s'. */ -struct SavingSourceAdapter : Source -{ - Source & orig; - string s; - SavingSourceAdapter(Source & orig) : orig(orig) { } - size_t read(unsigned char * data, size_t len) - { - size_t n = orig.read(data, len); - s.append((const char *) data, n); - return n; - } -}; - - -static void performOp(bool trusted, unsigned int clientVersion, - Source & from, Sink & to, unsigned int op) -{ - switch (op) { - -#if 0 - case wopQuit: { - /* Close the database. */ - store.reset((StoreAPI *) 0); - writeInt(1, to); - break; - } -#endif - - case wopIsValidPath: { - /* 'readStorePath' could raise an error leading to the connection - being closed. To be able to recover from an invalid path error, - call 'startWork' early, and do 'assertStorePath' afterwards so - that the 'Error' exception handler doesn't close the - connection. */ - Path path = readString(from); - startWork(); - assertStorePath(path); - bool result = store->isValidPath(path); - stopWork(); - writeInt(result, to); - break; - } - - case wopQueryValidPaths: { - PathSet paths = readStorePaths<PathSet>(from); - startWork(); - PathSet res = store->queryValidPaths(paths); - stopWork(); - writeStrings(res, to); - break; - } - - case wopHasSubstitutes: { - Path path = readStorePath(from); - startWork(); - PathSet res = store->querySubstitutablePaths(singleton<PathSet>(path)); - stopWork(); - writeInt(res.find(path) != res.end(), to); - break; - } - - case wopQuerySubstitutablePaths: { - PathSet paths = readStorePaths<PathSet>(from); - startWork(); - PathSet res = store->querySubstitutablePaths(paths); - stopWork(); - writeStrings(res, to); - break; - } - - case wopQueryPathHash: { - Path path = readStorePath(from); - startWork(); - Hash hash = store->queryPathHash(path); - stopWork(); - writeString(printHash(hash), to); - break; - } - - case wopQueryReferences: - case wopQueryReferrers: - case wopQueryValidDerivers: - case wopQueryDerivationOutputs: { - Path path = readStorePath(from); - startWork(); - PathSet paths; - if (op == wopQueryReferences) - store->queryReferences(path, paths); - else if (op == wopQueryReferrers) - store->queryReferrers(path, paths); - else if (op == wopQueryValidDerivers) - paths = store->queryValidDerivers(path); - else paths = store->queryDerivationOutputs(path); - stopWork(); - writeStrings(paths, to); - break; - } - - case wopQueryDerivationOutputNames: { - Path path = readStorePath(from); - startWork(); - StringSet names; - names = store->queryDerivationOutputNames(path); - stopWork(); - writeStrings(names, to); - break; - } - - case wopQueryDeriver: { - Path path = readStorePath(from); - startWork(); - Path deriver = store->queryDeriver(path); - stopWork(); - writeString(deriver, to); - break; - } - - case wopQueryPathFromHashPart: { - string hashPart = readString(from); - startWork(); - Path path = store->queryPathFromHashPart(hashPart); - stopWork(); - writeString(path, to); - break; - } - - case wopAddToStore: { - string baseName = readString(from); - bool fixed = readInt(from) == 1; /* obsolete */ - bool recursive = readInt(from) == 1; - string s = readString(from); - /* Compatibility hack. */ - if (!fixed) { - s = "sha256"; - recursive = true; - } - HashType hashAlgo = parseHashType(s); - - SavingSourceAdapter savedNAR(from); - RetrieveRegularNARSink savedRegular; - - if (recursive) { - /* Get the entire NAR dump from the client and save it to - a string so that we can pass it to - addToStoreFromDump(). */ - ParseSink sink; /* null sink; just parse the NAR */ - parseDump(sink, savedNAR); - } else - parseDump(savedRegular, from); - - startWork(); - if (!savedRegular.regular) throw Error("regular file expected"); - Path path = dynamic_cast<LocalStore *>(store.get()) - ->addToStoreFromDump(recursive ? savedNAR.s : savedRegular.s, baseName, recursive, hashAlgo); - stopWork(); - - writeString(path, to); - break; - } - - case wopAddTextToStore: { - string suffix = readString(from); - string s = readString(from); - PathSet refs = readStorePaths<PathSet>(from); - startWork(); - Path path = store->addTextToStore(suffix, s, refs); - stopWork(); - writeString(path, to); - break; - } - - case wopExportPath: { - Path path = readStorePath(from); - bool sign = readInt(from) == 1; - startWork(); - TunnelSink sink(to); - store->exportPath(path, sign, sink); - stopWork(); - writeInt(1, to); - break; - } - - case wopImportPaths: { - startWork(); - TunnelSource source(from); - Paths paths = store->importPaths(true, source); - stopWork(); - writeStrings(paths, to); - break; - } - - case wopBuildPaths: { - PathSet drvs = readStorePaths<PathSet>(from); - startWork(); - store->buildPaths(drvs); - stopWork(); - writeInt(1, to); - break; - } - - case wopEnsurePath: { - Path path = readStorePath(from); - startWork(); - store->ensurePath(path); - stopWork(); - writeInt(1, to); - break; - } - - case wopAddTempRoot: { - Path path = readStorePath(from); - startWork(); - store->addTempRoot(path); - stopWork(); - writeInt(1, to); - break; - } - - case wopAddIndirectRoot: { - Path path = absPath(readString(from)); - startWork(); - store->addIndirectRoot(path); - stopWork(); - writeInt(1, to); - break; - } - - case wopSyncWithGC: { - startWork(); - store->syncWithGC(); - stopWork(); - writeInt(1, to); - break; - } - - case wopFindRoots: { - startWork(); - Roots roots = store->findRoots(); - stopWork(); - writeInt(roots.size(), to); - for (Roots::iterator i = roots.begin(); i != roots.end(); ++i) { - writeString(i->first, to); - writeString(i->second, to); - } - break; - } - - case wopCollectGarbage: { - GCOptions options; - options.action = (GCOptions::GCAction) readInt(from); - options.pathsToDelete = readStorePaths<PathSet>(from); - options.ignoreLiveness = readInt(from); - options.maxFreed = readLongLong(from); - readInt(from); // obsolete field - if (GET_PROTOCOL_MINOR(clientVersion) >= 5) { - /* removed options */ - readInt(from); - readInt(from); - } - - GCResults results; - - startWork(); - if (options.ignoreLiveness) - throw Error("you are not allowed to ignore liveness"); - store->collectGarbage(options, results); - stopWork(); - - writeStrings(results.paths, to); - writeLongLong(results.bytesFreed, to); - writeLongLong(0, to); // obsolete - - break; - } - - case wopSetOptions: { - settings.keepFailed = readInt(from) != 0; - settings.keepGoing = readInt(from) != 0; - settings.set("build-fallback", readInt(from) ? "true" : "false"); - verbosity = (Verbosity) readInt(from); - settings.set("build-max-jobs", int2String(readInt(from))); - settings.set("build-max-silent-time", int2String(readInt(from))); - if (GET_PROTOCOL_MINOR(clientVersion) >= 2) - settings.useBuildHook = readInt(from) != 0; - if (GET_PROTOCOL_MINOR(clientVersion) >= 4) { - settings.buildVerbosity = (Verbosity) readInt(from); - logType = (LogType) readInt(from); - settings.printBuildTrace = readInt(from) != 0; - } - if (GET_PROTOCOL_MINOR(clientVersion) >= 6) - settings.set("build-cores", int2String(readInt(from))); - if (GET_PROTOCOL_MINOR(clientVersion) >= 10) - settings.set("build-use-substitutes", readInt(from) ? "true" : "false"); - if (GET_PROTOCOL_MINOR(clientVersion) >= 12) { - unsigned int n = readInt(from); - for (unsigned int i = 0; i < n; i++) { - string name = readString(from); - string value = readString(from); - if (name == "build-timeout" || name == "use-ssh-substituter") - settings.set(name, value); - else - settings.set(trusted ? name : "untrusted-" + name, value); - } - } - settings.update(); - startWork(); - stopWork(); - break; - } - - case wopQuerySubstitutablePathInfo: { - Path path = absPath(readString(from)); - startWork(); - SubstitutablePathInfos infos; - store->querySubstitutablePathInfos(singleton<PathSet>(path), infos); - stopWork(); - SubstitutablePathInfos::iterator i = infos.find(path); - if (i == infos.end()) - writeInt(0, to); - else { - writeInt(1, to); - writeString(i->second.deriver, to); - writeStrings(i->second.references, to); - writeLongLong(i->second.downloadSize, to); - if (GET_PROTOCOL_MINOR(clientVersion) >= 7) - writeLongLong(i->second.narSize, to); - } - break; - } - - case wopQuerySubstitutablePathInfos: { - PathSet paths = readStorePaths<PathSet>(from); - startWork(); - SubstitutablePathInfos infos; - store->querySubstitutablePathInfos(paths, infos); - stopWork(); - writeInt(infos.size(), to); - foreach (SubstitutablePathInfos::iterator, i, infos) { - writeString(i->first, to); - writeString(i->second.deriver, to); - writeStrings(i->second.references, to); - writeLongLong(i->second.downloadSize, to); - writeLongLong(i->second.narSize, to); - } - break; - } - - case wopQueryAllValidPaths: { - startWork(); - PathSet paths = store->queryAllValidPaths(); - stopWork(); - writeStrings(paths, to); - break; - } - - case wopQueryFailedPaths: { - startWork(); - PathSet paths = store->queryFailedPaths(); - stopWork(); - writeStrings(paths, to); - break; - } - - case wopClearFailedPaths: { - PathSet paths = readStrings<PathSet>(from); - startWork(); - store->clearFailedPaths(paths); - stopWork(); - writeInt(1, to); - break; - } - - case wopQueryPathInfo: { - Path path = readStorePath(from); - startWork(); - ValidPathInfo info = store->queryPathInfo(path); - stopWork(); - writeString(info.deriver, to); - writeString(printHash(info.hash), to); - writeStrings(info.references, to); - writeInt(info.registrationTime, to); - writeLongLong(info.narSize, to); - break; - } - - default: - throw Error(format("invalid operation %1%") % op); - } -} - - -static void processConnection(bool trusted) -{ - canSendStderr = false; - myPid = getpid(); - _writeToStderr = tunnelStderr; - -#ifdef HAVE_HUP_NOTIFICATION - /* Allow us to receive SIGPOLL for events on the client socket. */ - setSigPollAction(false); - if (fcntl(from.fd, F_SETOWN, getpid()) == -1) - throw SysError("F_SETOWN"); - if (fcntl(from.fd, F_SETFL, fcntl(from.fd, F_GETFL, 0) | O_ASYNC) == -1) - throw SysError("F_SETFL"); -#endif - - /* Exchange the greeting. */ - unsigned int magic = readInt(from); - if (magic != WORKER_MAGIC_1) throw Error("protocol mismatch"); - writeInt(WORKER_MAGIC_2, to); - writeInt(PROTOCOL_VERSION, to); - to.flush(); - unsigned int clientVersion = readInt(from); - - if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from)) - setAffinityTo(readInt(from)); - - bool reserveSpace = true; - if (GET_PROTOCOL_MINOR(clientVersion) >= 11) - reserveSpace = readInt(from) != 0; - - /* Send startup error messages to the client. */ - startWork(); - - try { - - /* If we can't accept clientVersion, then throw an error - *here* (not above). */ - -#if 0 - /* Prevent users from doing something very dangerous. */ - if (geteuid() == 0 && - querySetting("build-users-group", "") == "") - throw Error("if you run `nix-daemon' as root, then you MUST set `build-users-group'!"); -#endif - - /* Open the store. */ - store = std::shared_ptr<StoreAPI>(new LocalStore(reserveSpace)); - - stopWork(); - to.flush(); - - } catch (Error & e) { - stopWork(false, e.msg()); - to.flush(); - return; - } - - /* Process client requests. */ - unsigned int opCount = 0; - - while (true) { - WorkerOp op; - try { - op = (WorkerOp) readInt(from); - } catch (EndOfFile & e) { - break; - } - - opCount++; - - try { - performOp(trusted, clientVersion, from, to, op); - } catch (Error & e) { - /* If we're not in a state where we can send replies, then - something went wrong processing the input of the - client. This can happen especially if I/O errors occur - during addTextToStore() / importPath(). If that - happens, just send the error message and exit. */ - bool errorAllowed = canSendStderr; - if (!errorAllowed) printMsg(lvlError, format("error processing client input: %1%") % e.msg()); - stopWork(false, e.msg(), GET_PROTOCOL_MINOR(clientVersion) >= 8 ? e.status : 0); - if (!errorAllowed) break; - } catch (std::bad_alloc & e) { - if (canSendStderr) - stopWork(false, "Nix daemon out of memory", GET_PROTOCOL_MINOR(clientVersion) >= 8 ? 1 : 0); - throw; - } - - to.flush(); - - assert(!canSendStderr); - }; - - printMsg(lvlError, format("%1% operations") % opCount); -} - - -static void sigChldHandler(int sigNo) -{ - /* Reap all dead children. */ - while (waitpid(-1, 0, WNOHANG) > 0) ; -} - - -static void setSigChldAction(bool autoReap) -{ - struct sigaction act, oact; - act.sa_handler = autoReap ? sigChldHandler : SIG_DFL; - sigfillset(&act.sa_mask); - act.sa_flags = 0; - if (sigaction(SIGCHLD, &act, &oact)) - throw SysError("setting SIGCHLD handler"); -} - - -#define SD_LISTEN_FDS_START 3 - - -static void daemonLoop() -{ - /* Get rid of children automatically; don't let them become - zombies. */ - setSigChldAction(true); - - AutoCloseFD fdSocket; - - /* Handle socket-based activation by systemd. */ - if (getEnv("LISTEN_FDS") != "") { - if (getEnv("LISTEN_PID") != int2String(getpid()) || getEnv("LISTEN_FDS") != "1") - throw Error("unexpected systemd environment variables"); - fdSocket = SD_LISTEN_FDS_START; - } - - /* Otherwise, create and bind to a Unix domain socket. */ - else { - - /* Create and bind to a Unix domain socket. */ - fdSocket = socket(PF_UNIX, SOCK_STREAM, 0); - if (fdSocket == -1) - throw SysError("cannot create Unix domain socket"); - - string socketPath = settings.nixDaemonSocketFile; - - createDirs(dirOf(socketPath)); - - /* Urgh, sockaddr_un allows path names of only 108 characters. - So chdir to the socket directory so that we can pass a - relative path name. */ - chdir(dirOf(socketPath).c_str()); - Path socketPathRel = "./" + baseNameOf(socketPath); - - struct sockaddr_un addr; - addr.sun_family = AF_UNIX; - if (socketPathRel.size() >= sizeof(addr.sun_path)) - throw Error(format("socket path `%1%' is too long") % socketPathRel); - strcpy(addr.sun_path, socketPathRel.c_str()); - - unlink(socketPath.c_str()); - - /* Make sure that the socket is created with 0666 permission - (everybody can connect --- provided they have access to the - directory containing the socket). */ - mode_t oldMode = umask(0111); - int res = bind(fdSocket, (struct sockaddr *) &addr, sizeof(addr)); - umask(oldMode); - if (res == -1) - throw SysError(format("cannot bind to socket `%1%'") % socketPath); - - chdir("/"); /* back to the root */ - - if (listen(fdSocket, 5) == -1) - throw SysError(format("cannot listen on socket `%1%'") % socketPath); - } - - closeOnExec(fdSocket); - - /* Loop accepting connections. */ - while (1) { - - try { - /* Important: the server process *cannot* open the SQLite - database, because it doesn't like forks very much. */ - assert(!store); - - /* Accept a connection. */ - struct sockaddr_un remoteAddr; - socklen_t remoteAddrLen = sizeof(remoteAddr); - - AutoCloseFD remote = accept(fdSocket, - (struct sockaddr *) &remoteAddr, &remoteAddrLen); - checkInterrupt(); - if (remote == -1) { - if (errno == EINTR) - continue; - else - throw SysError("accepting connection"); - } - - closeOnExec(remote); - - /* Get the identity of the caller, if possible. */ - uid_t clientUid = -1; - pid_t clientPid = -1; - bool trusted = false; - -#if defined(SO_PEERCRED) - ucred cred; - socklen_t credLen = sizeof(cred); - if (getsockopt(remote, SOL_SOCKET, SO_PEERCRED, &cred, &credLen) != -1) { - clientPid = cred.pid; - clientUid = cred.uid; - if (clientUid == 0) trusted = true; - } -#endif - - printMsg(lvlInfo, format("accepted connection from pid %1%, uid %2%") % clientPid % clientUid); - - /* Fork a child to handle the connection. */ - pid_t child; - child = fork(); - - switch (child) { - - case -1: - throw SysError("unable to fork"); - - case 0: - try { /* child */ - - /* Background the daemon. */ - if (setsid() == -1) - throw SysError(format("creating a new session")); - - /* Restore normal handling of SIGCHLD. */ - setSigChldAction(false); - - /* For debugging, stuff the pid into argv[1]. */ - if (clientPid != -1 && argvSaved[1]) { - string processName = int2String(clientPid); - strncpy(argvSaved[1], processName.c_str(), strlen(argvSaved[1])); - } - - /* Handle the connection. */ - from.fd = remote; - to.fd = remote; - processConnection(trusted); - - } catch (std::exception & e) { - writeToStderr("unexpected Nix daemon error: " + string(e.what()) + "\n"); - } - exit(0); - } - - } catch (Interrupted & e) { - throw; - } catch (Error & e) { - printMsg(lvlError, format("error processing connection: %1%") % e.msg()); - } - } -} - - -void run(Strings args) -{ - for (Strings::iterator i = args.begin(); i != args.end(); ) { - string arg = *i++; - if (arg == "--daemon") /* ignored for backwards compatibility */; - } - - chdir("/"); - daemonLoop(); -} - - -void printHelp() -{ - showManPage("nix-daemon"); -} - - -string programId = "nix-daemon"; diff --git a/src/nix-env/local.mk b/src/nix-env/local.mk deleted file mode 100644 index e80719cd76..0000000000 --- a/src/nix-env/local.mk +++ /dev/null @@ -1,7 +0,0 @@ -programs += nix-env - -nix-env_DIR := $(d) - -nix-env_SOURCES := $(wildcard $(d)/*.cc) - -nix-env_LIBS = libexpr libmain libstore libutil libformat diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc deleted file mode 100644 index 8a80526347..0000000000 --- a/src/nix-env/nix-env.cc +++ /dev/null @@ -1,1441 +0,0 @@ -#include "profiles.hh" -#include "names.hh" -#include "globals.hh" -#include "misc.hh" -#include "shared.hh" -#include "eval.hh" -#include "get-drvs.hh" -#include "attr-path.hh" -#include "common-opts.hh" -#include "xml-writer.hh" -#include "store-api.hh" -#include "user-env.hh" -#include "util.hh" -#include "value-to-json.hh" - -#include <cerrno> -#include <ctime> -#include <algorithm> -#include <iostream> -#include <sstream> - -#include <sys/types.h> -#include <sys/stat.h> -#include <unistd.h> - - -using namespace nix; -using std::cout; - - -typedef enum { - srcNixExprDrvs, - srcNixExprs, - srcStorePaths, - srcProfile, - srcAttrPath, - srcUnknown -} InstallSourceType; - - -struct InstallSourceInfo -{ - InstallSourceType type; - Path nixExprPath; /* for srcNixExprDrvs, srcNixExprs */ - Path profile; /* for srcProfile */ - string systemFilter; /* for srcNixExprDrvs */ - Bindings autoArgs; -}; - - -struct Globals -{ - InstallSourceInfo instSource; - Path profile; - EvalState state; - bool dryRun; - bool preserveInstalled; - bool removeAll; - string forceName; - bool prebuiltOnly; -}; - - -typedef void (* Operation) (Globals & globals, - Strings args, Strings opFlags, Strings opArgs); - - -void printHelp() -{ - showManPage("nix-env"); -} - - -static string needArg(Strings::iterator & i, - Strings & args, const string & arg) -{ - if (i == args.end()) throw UsageError( - format("`%1%' requires an argument") % arg); - return *i++; -} - - -static bool parseInstallSourceOptions(Globals & globals, - Strings::iterator & i, Strings & args, const string & arg) -{ - if (arg == "--from-expression" || arg == "-E") - globals.instSource.type = srcNixExprs; - else if (arg == "--from-profile") { - globals.instSource.type = srcProfile; - globals.instSource.profile = needArg(i, args, arg); - } - else if (arg == "--attr" || arg == "-A") - globals.instSource.type = srcAttrPath; - else return false; - return true; -} - - -static bool isNixExpr(const Path & path, struct stat & st) -{ - return S_ISREG(st.st_mode) || (S_ISDIR(st.st_mode) && pathExists(path + "/default.nix")); -} - - -static void getAllExprs(EvalState & state, - const Path & path, StringSet & attrs, Value & v) -{ - Strings names = readDirectory(path); - StringSet namesSorted(names.begin(), names.end()); - - foreach (StringSet::iterator, i, namesSorted) { - /* Ignore the manifest.nix used by profiles. This is - necessary to prevent it from showing up in channels (which - are implemented using profiles). */ - if (*i == "manifest.nix") continue; - - Path path2 = path + "/" + *i; - - struct stat st; - if (stat(path2.c_str(), &st) == -1) - continue; // ignore dangling symlinks in ~/.nix-defexpr - - if (isNixExpr(path2, st) && (!S_ISREG(st.st_mode) || hasSuffix(path2, ".nix"))) { - /* Strip off the `.nix' filename suffix (if applicable), - otherwise the attribute cannot be selected with the - `-A' option. Useful if you want to stick a Nix - expression directly in ~/.nix-defexpr. */ - string attrName = *i; - if (hasSuffix(attrName, ".nix")) - attrName = string(attrName, 0, attrName.size() - 4); - if (attrs.find(attrName) != attrs.end()) { - printMsg(lvlError, format("warning: name collision in input Nix expressions, skipping `%1%'") % path2); - continue; - } - attrs.insert(attrName); - /* Load the expression on demand. */ - Value & vFun(*state.allocValue()); - Value & vArg(*state.allocValue()); - state.getBuiltin("import", vFun); - mkString(vArg, path2); - mkApp(*state.allocAttr(v, state.symbols.create(attrName)), vFun, vArg); - } - else if (S_ISDIR(st.st_mode)) - /* `path2' is a directory (with no default.nix in it); - recurse into it. */ - getAllExprs(state, path2, attrs, v); - } -} - - -static void loadSourceExpr(EvalState & state, const Path & path, Value & v) -{ - struct stat st; - if (stat(path.c_str(), &st) == -1) - throw SysError(format("getting information about `%1%'") % path); - - if (isNixExpr(path, st)) { - state.evalFile(path, v); - return; - } - - /* The path is a directory. Put the Nix expressions in the - directory in a set, with the file name of each expression as - the attribute name. Recurse into subdirectories (but keep the - set flat, not nested, to make it easier for a user to have a - ~/.nix-defexpr directory that includes some system-wide - directory). */ - if (S_ISDIR(st.st_mode)) { - state.mkAttrs(v, 16); - state.mkList(*state.allocAttr(v, state.symbols.create("_combineChannels")), 0); - StringSet attrs; - getAllExprs(state, path, attrs, v); - v.attrs->sort(); - } -} - - -static void loadDerivations(EvalState & state, Path nixExprPath, - string systemFilter, Bindings & autoArgs, - const string & pathPrefix, DrvInfos & elems) -{ - Value vRoot; - loadSourceExpr(state, nixExprPath, vRoot); - - Value & v(*findAlongAttrPath(state, pathPrefix, autoArgs, vRoot)); - - getDerivations(state, v, pathPrefix, autoArgs, elems, true); - - /* Filter out all derivations not applicable to the current - system. */ - for (DrvInfos::iterator i = elems.begin(), j; i != elems.end(); i = j) { - j = i; j++; - if (systemFilter != "*" && i->system != systemFilter) - elems.erase(i); - } -} - - -static Path getHomeDir() -{ - Path homeDir(getEnv("HOME", "")); - if (homeDir == "") throw Error("HOME environment variable not set"); - return homeDir; -} - - -static Path getDefNixExprPath() -{ - return getHomeDir() + "/.nix-defexpr"; -} - - -static int getPriority(EvalState & state, DrvInfo & drv) -{ - return drv.queryMetaInt("priority", 0); -} - - -static int comparePriorities(EvalState & state, DrvInfo & drv1, DrvInfo & drv2) -{ - return getPriority(state, drv2) - getPriority(state, drv1); -} - - -// FIXME: this function is rather slow since it checks a single path -// at a time. -static bool isPrebuilt(EvalState & state, DrvInfo & elem) -{ - Path path = elem.queryOutPath(); - if (store->isValidPath(path)) return true; - PathSet ps = store->querySubstitutablePaths(singleton<PathSet>(path)); - return ps.find(path) != ps.end(); -} - - -static void checkSelectorUse(DrvNames & selectors) -{ - /* Check that all selectors have been used. */ - foreach (DrvNames::iterator, i, selectors) - if (i->hits == 0 && i->fullName != "*") - throw Error(format("selector `%1%' matches no derivations") % i->fullName); -} - - -static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems, - const Strings & args, bool newestOnly) -{ - DrvNames selectors = drvNamesFromArgs(args); - if (selectors.empty()) - selectors.push_back(DrvName("*")); - - DrvInfos elems; - set<unsigned int> done; - - foreach (DrvNames::iterator, i, selectors) { - typedef list<std::pair<DrvInfo, unsigned int> > Matches; - Matches matches; - unsigned int n = 0; - for (DrvInfos::const_iterator j = allElems.begin(); - j != allElems.end(); ++j, ++n) - { - DrvName drvName(j->name); - if (i->matches(drvName)) { - i->hits++; - matches.push_back(std::pair<DrvInfo, unsigned int>(*j, n)); - } - } - - /* If `newestOnly', if a selector matches multiple derivations - with the same name, pick the one matching the current - system. If there are still multiple derivations, pick the - one with the highest priority. If there are still multiple - derivations, pick the one with the highest version. - Finally, if there are still multiple derivations, - arbitrarily pick the first one. */ - if (newestOnly) { - - /* Map from package names to derivations. */ - typedef map<string, std::pair<DrvInfo, unsigned int> > Newest; - Newest newest; - StringSet multiple; - - for (Matches::iterator j = matches.begin(); j != matches.end(); ++j) { - DrvName drvName(j->first.name); - int d = 1; - - Newest::iterator k = newest.find(drvName.name); - - if (k != newest.end()) { - d = j->first.system == k->second.first.system ? 0 : - j->first.system == settings.thisSystem ? 1 : - k->second.first.system == settings.thisSystem ? -1 : 0; - if (d == 0) - d = comparePriorities(state, j->first, k->second.first); - if (d == 0) - d = compareVersions(drvName.version, DrvName(k->second.first.name).version); - } - - if (d > 0) { - newest.erase(drvName.name); - newest.insert(Newest::value_type(drvName.name, *j)); - multiple.erase(j->first.name); - } else if (d == 0) { - multiple.insert(j->first.name); - } - } - - matches.clear(); - for (Newest::iterator j = newest.begin(); j != newest.end(); ++j) { - if (multiple.find(j->second.first.name) != multiple.end()) - printMsg(lvlInfo, - format("warning: there are multiple derivations named `%1%'; using the first one") - % j->second.first.name); - matches.push_back(j->second); - } - } - - /* Insert only those elements in the final list that we - haven't inserted before. */ - for (Matches::iterator j = matches.begin(); j != matches.end(); ++j) - if (done.find(j->second) == done.end()) { - done.insert(j->second); - elems.push_back(j->first); - } - } - - checkSelectorUse(selectors); - - return elems; -} - - -static bool isPath(const string & s) -{ - return s.find('/') != string::npos; -} - - -static void queryInstSources(EvalState & state, - InstallSourceInfo & instSource, const Strings & args, - DrvInfos & elems, bool newestOnly) -{ - InstallSourceType type = instSource.type; - if (type == srcUnknown && args.size() > 0 && isPath(args.front())) - type = srcStorePaths; - - switch (type) { - - /* Get the available user environment elements from the - derivations specified in a Nix expression, including only - those with names matching any of the names in `args'. */ - case srcUnknown: - case srcNixExprDrvs: { - - /* Load the derivations from the (default or specified) - Nix expression. */ - DrvInfos allElems; - loadDerivations(state, instSource.nixExprPath, - instSource.systemFilter, instSource.autoArgs, "", allElems); - - elems = filterBySelector(state, allElems, args, newestOnly); - - break; - } - - /* Get the available user environment elements from the Nix - expressions specified on the command line; these should be - functions that take the default Nix expression file as - argument, e.g., if the file is `./foo.nix', then the - argument `x: x.bar' is equivalent to `(x: x.bar) - (import ./foo.nix)' = `(import ./foo.nix).bar'. */ - case srcNixExprs: { - - Value vArg; - loadSourceExpr(state, instSource.nixExprPath, vArg); - - foreach (Strings::const_iterator, i, args) { - Expr * eFun = state.parseExprFromString(*i, absPath(".")); - Value vFun, vTmp; - state.eval(eFun, vFun); - mkApp(vTmp, vFun, vArg); - getDerivations(state, vTmp, "", instSource.autoArgs, elems, true); - } - - break; - } - - /* The available user environment elements are specified as a - list of store paths (which may or may not be - derivations). */ - case srcStorePaths: { - - foreach (Strings::const_iterator, i, args) { - Path path = followLinksToStorePath(*i); - - string name = baseNameOf(path); - string::size_type dash = name.find('-'); - if (dash != string::npos) - name = string(name, dash + 1); - - DrvInfo elem(state, name, "", "", 0); - - if (isDerivation(path)) { - elem.setDrvPath(path); - elem.setOutPath(findOutput(derivationFromPath(*store, path), "out")); - if (name.size() >= drvExtension.size() && - string(name, name.size() - drvExtension.size()) == drvExtension) - name = string(name, 0, name.size() - drvExtension.size()); - } - else elem.setOutPath(path); - - elems.push_back(elem); - } - - break; - } - - /* Get the available user environment elements from another - user environment. These are then filtered as in the - `srcNixExprDrvs' case. */ - case srcProfile: { - elems = filterBySelector(state, - queryInstalled(state, instSource.profile), - args, newestOnly); - break; - } - - case srcAttrPath: { - Value vRoot; - loadSourceExpr(state, instSource.nixExprPath, vRoot); - foreach (Strings::const_iterator, i, args) { - Value & v(*findAlongAttrPath(state, *i, instSource.autoArgs, vRoot)); - getDerivations(state, v, "", instSource.autoArgs, elems, true); - } - break; - } - } -} - - -static void printMissing(EvalState & state, DrvInfos & elems) -{ - PathSet targets; - foreach (DrvInfos::iterator, i, elems) { - Path drvPath = i->queryDrvPath(); - if (drvPath != "") - targets.insert(drvPath); - else - targets.insert(i->queryOutPath()); - } - - printMissing(*store, targets); -} - - -static bool keep(DrvInfo & drv) -{ - return drv.queryMetaBool("keep", false); -} - - -static void installDerivations(Globals & globals, - const Strings & args, const Path & profile) -{ - debug(format("installing derivations")); - - /* Get the set of user environment elements to be installed. */ - DrvInfos newElems, newElemsTmp; - queryInstSources(globals.state, globals.instSource, args, newElemsTmp, true); - - /* If --prebuilt-only is given, filter out source-only packages. */ - foreach (DrvInfos::iterator, i, newElemsTmp) - if (!globals.prebuiltOnly || isPrebuilt(globals.state, *i)) - newElems.push_back(*i); - - StringSet newNames; - for (DrvInfos::iterator i = newElems.begin(); i != newElems.end(); ++i) { - /* `forceName' is a hack to get package names right in some - one-click installs, namely those where the name used in the - path is not the one we want (e.g., `java-front' versus - `java-front-0.9pre15899'). */ - if (globals.forceName != "") - i->name = globals.forceName; - newNames.insert(DrvName(i->name).name); - } - - - while (true) { - string lockToken = optimisticLockProfile(profile); - - DrvInfos allElems(newElems); - - /* Add in the already installed derivations, unless they have - the same name as a to-be-installed element. */ - if (!globals.removeAll) { - DrvInfos installedElems = queryInstalled(globals.state, profile); - - foreach (DrvInfos::iterator, i, installedElems) { - DrvName drvName(i->name); - if (!globals.preserveInstalled && - newNames.find(drvName.name) != newNames.end() && - !keep(*i)) - printMsg(lvlInfo, format("replacing old `%1%'") % i->name); - else - allElems.push_back(*i); - } - - foreach (DrvInfos::iterator, i, newElems) - printMsg(lvlInfo, format("installing `%1%'") % i->name); - } - - printMissing(globals.state, newElems); - - if (globals.dryRun) return; - - if (createUserEnv(globals.state, allElems, - profile, settings.envKeepDerivations, lockToken)) break; - } -} - - -static void opInstall(Globals & globals, - Strings args, Strings opFlags, Strings opArgs) -{ - for (Strings::iterator i = opFlags.begin(); i != opFlags.end(); ) { - string arg = *i++; - if (parseInstallSourceOptions(globals, i, opFlags, arg)) ; - else if (arg == "--preserve-installed" || arg == "-P") - globals.preserveInstalled = true; - else if (arg == "--remove-all" || arg == "-r") - globals.removeAll = true; - else throw UsageError(format("unknown flag `%1%'") % arg); - } - - installDerivations(globals, opArgs, globals.profile); -} - - -typedef enum { utLt, utLeq, utEq, utAlways } UpgradeType; - - -static void upgradeDerivations(Globals & globals, - const Strings & args, UpgradeType upgradeType) -{ - debug(format("upgrading derivations")); - - /* Upgrade works as follows: we take all currently installed - derivations, and for any derivation matching any selector, look - for a derivation in the input Nix expression that has the same - name and a higher version number. */ - - while (true) { - string lockToken = optimisticLockProfile(globals.profile); - - DrvInfos installedElems = queryInstalled(globals.state, globals.profile); - - /* Fetch all derivations from the input file. */ - DrvInfos availElems; - queryInstSources(globals.state, globals.instSource, args, availElems, false); - - /* Go through all installed derivations. */ - DrvInfos newElems; - foreach (DrvInfos::iterator, i, installedElems) { - DrvName drvName(i->name); - - try { - - if (keep(*i)) { - newElems.push_back(*i); - continue; - } - - /* Find the derivation in the input Nix expression - with the same name that satisfies the version - constraints specified by upgradeType. If there are - multiple matches, take the one with the highest - priority. If there are still multiple matches, - take the one with the highest version. */ - DrvInfos::iterator bestElem = availElems.end(); - DrvName bestName; - foreach (DrvInfos::iterator, j, availElems) { - DrvName newName(j->name); - if (newName.name == drvName.name) { - int d = comparePriorities(globals.state, *i, *j); - if (d == 0) d = compareVersions(drvName.version, newName.version); - if ((upgradeType == utLt && d < 0) || - (upgradeType == utLeq && d <= 0) || - (upgradeType == utEq && d == 0) || - upgradeType == utAlways) - { - int d2 = -1; - if (bestElem != availElems.end()) { - d2 = comparePriorities(globals.state, *bestElem, *j); - if (d2 == 0) d2 = compareVersions(bestName.version, newName.version); - } - if (d2 < 0 && (!globals.prebuiltOnly || isPrebuilt(globals.state, *j))) { - bestElem = j; - bestName = newName; - } - } - } - } - - if (bestElem != availElems.end() && - i->queryOutPath() != - bestElem->queryOutPath()) - { - printMsg(lvlInfo, - format("upgrading `%1%' to `%2%'") - % i->name % bestElem->name); - newElems.push_back(*bestElem); - } else newElems.push_back(*i); - - } catch (Error & e) { - e.addPrefix(format("while trying to find an upgrade for `%1%':\n") % i->name); - throw; - } - } - - printMissing(globals.state, newElems); - - if (globals.dryRun) return; - - if (createUserEnv(globals.state, newElems, - globals.profile, settings.envKeepDerivations, lockToken)) break; - } -} - - -static void opUpgrade(Globals & globals, - Strings args, Strings opFlags, Strings opArgs) -{ - UpgradeType upgradeType = utLt; - for (Strings::iterator i = opFlags.begin(); i != opFlags.end(); ) { - string arg = *i++; - if (parseInstallSourceOptions(globals, i, opFlags, arg)) ; - else if (arg == "--lt") upgradeType = utLt; - else if (arg == "--leq") upgradeType = utLeq; - else if (arg == "--eq") upgradeType = utEq; - else if (arg == "--always") upgradeType = utAlways; - else throw UsageError(format("unknown flag `%1%'") % arg); - } - - upgradeDerivations(globals, opArgs, upgradeType); -} - - -static void setMetaFlag(EvalState & state, DrvInfo & drv, - const string & name, const string & value) -{ - Value * v = state.allocValue(); - mkString(*v, value.c_str()); - drv.setMeta(name, v); -} - - -static void opSetFlag(Globals & globals, - Strings args, Strings opFlags, Strings opArgs) -{ - if (opFlags.size() > 0) - throw UsageError(format("unknown flag `%1%'") % opFlags.front()); - if (opArgs.size() < 2) - throw UsageError("not enough arguments to `--set-flag'"); - - Strings::iterator arg = opArgs.begin(); - string flagName = *arg++; - string flagValue = *arg++; - DrvNames selectors = drvNamesFromArgs(Strings(arg, opArgs.end())); - - while (true) { - string lockToken = optimisticLockProfile(globals.profile); - - DrvInfos installedElems = queryInstalled(globals.state, globals.profile); - - /* Update all matching derivations. */ - foreach (DrvInfos::iterator, i, installedElems) { - DrvName drvName(i->name); - foreach (DrvNames::iterator, j, selectors) - if (j->matches(drvName)) { - printMsg(lvlInfo, format("setting flag on `%1%'") % i->name); - j->hits++; - setMetaFlag(globals.state, *i, flagName, flagValue); - break; - } - } - - checkSelectorUse(selectors); - - /* Write the new user environment. */ - if (createUserEnv(globals.state, installedElems, - globals.profile, settings.envKeepDerivations, lockToken)) break; - } -} - - -static void opSet(Globals & globals, - Strings args, Strings opFlags, Strings opArgs) -{ - for (Strings::iterator i = opFlags.begin(); i != opFlags.end(); ) { - string arg = *i++; - if (parseInstallSourceOptions(globals, i, opFlags, arg)) ; - else throw UsageError(format("unknown flag `%1%'") % arg); - } - - DrvInfos elems; - queryInstSources(globals.state, globals.instSource, opArgs, elems, true); - - if (elems.size() != 1) - throw Error("--set requires exactly one derivation"); - - DrvInfo & drv(elems.front()); - - if (drv.queryDrvPath() != "") { - PathSet paths = singleton<PathSet>(drv.queryDrvPath()); - printMissing(*store, paths); - if (globals.dryRun) return; - store->buildPaths(paths, globals.state.repair ? bmRepair : bmNormal); - } - else { - printMissing(*store, singleton<PathSet>(drv.queryOutPath())); - if (globals.dryRun) return; - store->ensurePath(drv.queryOutPath()); - } - - debug(format("switching to new user environment")); - Path generation = createGeneration(globals.profile, drv.queryOutPath()); - switchLink(globals.profile, generation); -} - - -static void uninstallDerivations(Globals & globals, Strings & selectors, - Path & profile) -{ - while (true) { - string lockToken = optimisticLockProfile(profile); - - DrvInfos installedElems = queryInstalled(globals.state, profile); - DrvInfos newElems; - - foreach (DrvInfos::iterator, i, installedElems) { - DrvName drvName(i->name); - bool found = false; - foreach (Strings::iterator, j, selectors) - /* !!! the repeated calls to followLinksToStorePath() - are expensive, should pre-compute them. */ - if ((isPath(*j) && i->queryOutPath() == followLinksToStorePath(*j)) - || DrvName(*j).matches(drvName)) - { - printMsg(lvlInfo, format("uninstalling `%1%'") % i->name); - found = true; - break; - } - if (!found) newElems.push_back(*i); - } - - if (globals.dryRun) return; - - if (createUserEnv(globals.state, newElems, - profile, settings.envKeepDerivations, lockToken)) break; - } -} - - -static void opUninstall(Globals & globals, - Strings args, Strings opFlags, Strings opArgs) -{ - if (opFlags.size() > 0) - throw UsageError(format("unknown flag `%1%'") % opFlags.front()); - uninstallDerivations(globals, opArgs, globals.profile); -} - - -static bool cmpChars(char a, char b) -{ - return toupper(a) < toupper(b); -} - - -static bool cmpElemByName(const DrvInfo & a, const DrvInfo & b) -{ - return lexicographical_compare( - a.name.begin(), a.name.end(), - b.name.begin(), b.name.end(), cmpChars); -} - - -typedef list<Strings> Table; - - -void printTable(Table & table) -{ - unsigned int nrColumns = table.size() > 0 ? table.front().size() : 0; - - vector<unsigned int> widths; - widths.resize(nrColumns); - - foreach (Table::iterator, i, table) { - assert(i->size() == nrColumns); - Strings::iterator j; - unsigned int column; - for (j = i->begin(), column = 0; j != i->end(); ++j, ++column) - if (j->size() > widths[column]) widths[column] = j->size(); - } - - foreach (Table::iterator, i, table) { - Strings::iterator j; - unsigned int column; - for (j = i->begin(), column = 0; j != i->end(); ++j, ++column) { - string s = *j; - replace(s.begin(), s.end(), '\n', ' '); - cout << s; - if (column < nrColumns - 1) - cout << string(widths[column] - s.size() + 2, ' '); - } - cout << std::endl; - } -} - - -/* This function compares the version of an element against the - versions in the given set of elements. `cvLess' means that only - lower versions are in the set, `cvEqual' means that at most an - equal version is in the set, and `cvGreater' means that there is at - least one element with a higher version in the set. `cvUnavail' - means that there are no elements with the same name in the set. */ - -typedef enum { cvLess, cvEqual, cvGreater, cvUnavail } VersionDiff; - -static VersionDiff compareVersionAgainstSet( - const DrvInfo & elem, const DrvInfos & elems, string & version) -{ - DrvName name(elem.name); - - VersionDiff diff = cvUnavail; - version = "?"; - - for (DrvInfos::const_iterator i = elems.begin(); i != elems.end(); ++i) { - DrvName name2(i->name); - if (name.name == name2.name) { - int d = compareVersions(name.version, name2.version); - if (d < 0) { - diff = cvGreater; - version = name2.version; - } - else if (diff != cvGreater && d == 0) { - diff = cvEqual; - version = name2.version; - } - else if (diff != cvGreater && diff != cvEqual && d > 0) { - diff = cvLess; - if (version == "" || compareVersions(version, name2.version) < 0) - version = name2.version; - } - } - } - - return diff; -} - - -static string colorString(const string & s) -{ - if (!isatty(STDOUT_FILENO)) return s; - return "\e[1;31m" + s + "\e[0m"; -} - - -static void queryJSON(Globals & globals, vector<DrvInfo> & elems) -{ - JSONObject topObj(cout); - foreach (vector<DrvInfo>::iterator, i, elems) { - topObj.attr(i->attrPath); - JSONObject pkgObj(cout); - - pkgObj.attr("name", i->name); - pkgObj.attr("system", i->system); - - pkgObj.attr("meta"); - JSONObject metaObj(cout); - StringSet metaNames = i->queryMetaNames(); - foreach (StringSet::iterator, j, metaNames) { - metaObj.attr(*j); - Value * v = i->queryMeta(*j); - if (!v) - printMsg(lvlError, format("derivation `%1%' has invalid meta attribute `%2%'") % i->name % *j); - else { - PathSet context; - printValueAsJSON(globals.state, true, *v, cout, context); - } - } - } -} - - -static void opQuery(Globals & globals, - Strings args, Strings opFlags, Strings opArgs) -{ - Strings remaining; - string attrPath; - - bool printStatus = false; - bool printName = true; - bool printAttrPath = false; - bool printSystem = false; - bool printDrvPath = false; - bool printOutPath = false; - bool printDescription = false; - bool printMeta = false; - bool compareVersions = false; - bool xmlOutput = false; - bool jsonOutput = false; - - enum { sInstalled, sAvailable } source = sInstalled; - - settings.readOnlyMode = true; /* makes evaluation a bit faster */ - - for (Strings::iterator i = args.begin(); i != args.end(); ) { - string arg = *i++; - if (arg == "--status" || arg == "-s") printStatus = true; - else if (arg == "--no-name") printName = false; - else if (arg == "--system") printSystem = true; - else if (arg == "--description") printDescription = true; - else if (arg == "--compare-versions" || arg == "-c") compareVersions = true; - else if (arg == "--drv-path") printDrvPath = true; - else if (arg == "--out-path") printOutPath = true; - else if (arg == "--meta") printMeta = true; - else if (arg == "--installed") source = sInstalled; - else if (arg == "--available" || arg == "-a") source = sAvailable; - else if (arg == "--xml") xmlOutput = true; - else if (arg == "--json") jsonOutput = true; - else if (arg == "--attr-path" || arg == "-P") printAttrPath = true; - else if (arg == "--attr" || arg == "-A") - attrPath = needArg(i, args, arg); - else if (arg[0] == '-') - throw UsageError(format("unknown flag `%1%'") % arg); - else remaining.push_back(arg); - } - - - /* Obtain derivation information from the specified source. */ - DrvInfos availElems, installedElems; - - if (source == sInstalled || compareVersions || printStatus) - installedElems = queryInstalled(globals.state, globals.profile); - - if (source == sAvailable || compareVersions) - loadDerivations(globals.state, globals.instSource.nixExprPath, - globals.instSource.systemFilter, globals.instSource.autoArgs, - attrPath, availElems); - - DrvInfos elems_ = filterBySelector(globals.state, - source == sInstalled ? installedElems : availElems, - remaining, false); - - DrvInfos & otherElems(source == sInstalled ? availElems : installedElems); - - - /* Sort them by name. */ - /* !!! */ - vector<DrvInfo> elems; - for (DrvInfos::iterator i = elems_.begin(); i != elems_.end(); ++i) - elems.push_back(*i); - sort(elems.begin(), elems.end(), cmpElemByName); - - - /* We only need to know the installed paths when we are querying - the status of the derivation. */ - PathSet installed; /* installed paths */ - - if (printStatus) { - for (DrvInfos::iterator i = installedElems.begin(); - i != installedElems.end(); ++i) - installed.insert(i->queryOutPath()); - } - - - /* Query which paths have substitutes. */ - PathSet validPaths, substitutablePaths; - if (printStatus || globals.prebuiltOnly) { - PathSet paths; - foreach (vector<DrvInfo>::iterator, i, elems) - try { - paths.insert(i->queryOutPath()); - } catch (AssertionError & e) { - printMsg(lvlTalkative, format("skipping derivation named `%1%' which gives an assertion failure") % i->name); - i->setFailed(); - } - validPaths = store->queryValidPaths(paths); - substitutablePaths = store->querySubstitutablePaths(paths); - } - - - /* Print the desired columns, or XML output. */ - if (jsonOutput) { - queryJSON(globals, elems); - return; - } - - Table table; - std::ostringstream dummy; - XMLWriter xml(true, *(xmlOutput ? &cout : &dummy)); - XMLOpenElement xmlRoot(xml, "items"); - - foreach (vector<DrvInfo>::iterator, i, elems) { - try { - if (i->hasFailed()) continue; - - startNest(nest, lvlDebug, format("outputting query result `%1%'") % i->attrPath); - - if (globals.prebuiltOnly && - validPaths.find(i->queryOutPath()) == validPaths.end() && - substitutablePaths.find(i->queryOutPath()) == substitutablePaths.end()) - continue; - - /* For table output. */ - Strings columns; - - /* For XML output. */ - XMLAttrs attrs; - - if (printStatus) { - Path outPath = i->queryOutPath(); - bool hasSubs = substitutablePaths.find(outPath) != substitutablePaths.end(); - bool isInstalled = installed.find(outPath) != installed.end(); - bool isValid = validPaths.find(outPath) != validPaths.end(); - if (xmlOutput) { - attrs["installed"] = isInstalled ? "1" : "0"; - attrs["valid"] = isValid ? "1" : "0"; - attrs["substitutable"] = hasSubs ? "1" : "0"; - } else - columns.push_back( - (string) (isInstalled ? "I" : "-") - + (isValid ? "P" : "-") - + (hasSubs ? "S" : "-")); - } - - if (xmlOutput) - attrs["attrPath"] = i->attrPath; - else if (printAttrPath) - columns.push_back(i->attrPath); - - if (xmlOutput) - attrs["name"] = i->name; - else if (printName) - columns.push_back(i->name); - - if (compareVersions) { - /* Compare this element against the versions of the - same named packages in either the set of available - elements, or the set of installed elements. !!! - This is O(N * M), should be O(N * lg M). */ - string version; - VersionDiff diff = compareVersionAgainstSet(*i, otherElems, version); - - char ch; - switch (diff) { - case cvLess: ch = '>'; break; - case cvEqual: ch = '='; break; - case cvGreater: ch = '<'; break; - case cvUnavail: ch = '-'; break; - default: abort(); - } - - if (xmlOutput) { - if (diff != cvUnavail) { - attrs["versionDiff"] = ch; - attrs["maxComparedVersion"] = version; - } - } else { - string column = (string) "" + ch + " " + version; - if (diff == cvGreater) column = colorString(column); - columns.push_back(column); - } - } - - if (xmlOutput) { - if (i->system != "") attrs["system"] = i->system; - } - else if (printSystem) - columns.push_back(i->system); - - if (printDrvPath) { - string drvPath = i->queryDrvPath(); - if (xmlOutput) { - if (drvPath != "") attrs["drvPath"] = drvPath; - } else - columns.push_back(drvPath == "" ? "-" : drvPath); - } - - if (printOutPath && !xmlOutput) { - DrvInfo::Outputs outputs = i->queryOutputs(); - string s; - foreach (DrvInfo::Outputs::iterator, j, outputs) { - if (!s.empty()) s += ';'; - if (j->first != "out") { s += j->first; s += "="; } - s += j->second; - } - columns.push_back(s); - } - - if (printDescription) { - string descr = i->queryMetaString("description"); - if (xmlOutput) { - if (descr != "") attrs["description"] = descr; - } else - columns.push_back(descr); - } - - if (xmlOutput) { - if (printOutPath || printMeta) { - XMLOpenElement item(xml, "item", attrs); - if (printOutPath) { - DrvInfo::Outputs outputs = i->queryOutputs(); - foreach (DrvInfo::Outputs::iterator, j, outputs) { - XMLAttrs attrs2; - attrs2["name"] = j->first; - attrs2["path"] = j->second; - xml.writeEmptyElement("output", attrs2); - } - } - if (printMeta) { - StringSet metaNames = i->queryMetaNames(); - foreach (StringSet::iterator, j, metaNames) { - XMLAttrs attrs2; - attrs2["name"] = *j; - Value * v = i->queryMeta(*j); - if (!v) - printMsg(lvlError, format("derivation `%1%' has invalid meta attribute `%2%'") % i->name % *j); - else { - if (v->type == tString) { - attrs2["type"] = "string"; - attrs2["value"] = v->string.s; - xml.writeEmptyElement("meta", attrs2); - } else if (v->type == tInt) { - attrs2["type"] = "int"; - attrs2["value"] = (format("%1%") % v->integer).str(); - xml.writeEmptyElement("meta", attrs2); - } else if (v->type == tBool) { - attrs2["type"] = "bool"; - attrs2["value"] = v->boolean ? "true" : "false"; - xml.writeEmptyElement("meta", attrs2); - } else if (v->type == tList) { - attrs2["type"] = "strings"; - XMLOpenElement m(xml, "meta", attrs2); - for (unsigned int j = 0; j < v->list.length; ++j) { - if (v->list.elems[j]->type != tString) continue; - XMLAttrs attrs3; - attrs3["value"] = v->list.elems[j]->string.s; - xml.writeEmptyElement("string", attrs3); - } - } - } - } - } - } else - xml.writeEmptyElement("item", attrs); - } else - table.push_back(columns); - - cout.flush(); - - } catch (AssertionError & e) { - printMsg(lvlTalkative, format("skipping derivation named `%1%' which gives an assertion failure") % i->name); - } catch (Error & e) { - e.addPrefix(format("while querying the derivation named `%1%':\n") % i->name); - throw; - } - } - - if (!xmlOutput) printTable(table); -} - - -static void opSwitchProfile(Globals & globals, - Strings args, Strings opFlags, Strings opArgs) -{ - if (opFlags.size() > 0) - throw UsageError(format("unknown flag `%1%'") % opFlags.front()); - if (opArgs.size() != 1) - throw UsageError(format("exactly one argument expected")); - - Path profile = absPath(opArgs.front()); - Path profileLink = getHomeDir() + "/.nix-profile"; - - switchLink(profileLink, profile); -} - - -static const int prevGen = -2; - - -static void switchGeneration(Globals & globals, int dstGen) -{ - PathLocks lock; - lockProfile(lock, globals.profile); - - int curGen; - Generations gens = findGenerations(globals.profile, curGen); - - Generation dst; - for (Generations::iterator i = gens.begin(); i != gens.end(); ++i) - if ((dstGen == prevGen && i->number < curGen) || - (dstGen >= 0 && i->number == dstGen)) - dst = *i; - - if (!dst) { - if (dstGen == prevGen) - throw Error(format("no generation older than the current (%1%) exists") - % curGen); - else - throw Error(format("generation %1% does not exist") % dstGen); - } - - printMsg(lvlInfo, format("switching from generation %1% to %2%") - % curGen % dst.number); - - if (globals.dryRun) return; - - switchLink(globals.profile, dst.path); -} - - -static void opSwitchGeneration(Globals & globals, - Strings args, Strings opFlags, Strings opArgs) -{ - if (opFlags.size() > 0) - throw UsageError(format("unknown flag `%1%'") % opFlags.front()); - if (opArgs.size() != 1) - throw UsageError(format("exactly one argument expected")); - - int dstGen; - if (!string2Int(opArgs.front(), dstGen)) - throw UsageError(format("expected a generation number")); - - switchGeneration(globals, dstGen); -} - - -static void opRollback(Globals & globals, - Strings args, Strings opFlags, Strings opArgs) -{ - if (opFlags.size() > 0) - throw UsageError(format("unknown flag `%1%'") % opFlags.front()); - if (opArgs.size() != 0) - throw UsageError(format("no arguments expected")); - - switchGeneration(globals, prevGen); -} - - -static void opListGenerations(Globals & globals, - Strings args, Strings opFlags, Strings opArgs) -{ - if (opFlags.size() > 0) - throw UsageError(format("unknown flag `%1%'") % opFlags.front()); - if (opArgs.size() != 0) - throw UsageError(format("no arguments expected")); - - PathLocks lock; - lockProfile(lock, globals.profile); - - int curGen; - Generations gens = findGenerations(globals.profile, curGen); - - for (Generations::iterator i = gens.begin(); i != gens.end(); ++i) { - tm t; - if (!localtime_r(&i->creationTime, &t)) throw Error("cannot convert time"); - cout << format("%|4| %|4|-%|02|-%|02| %|02|:%|02|:%|02| %||\n") - % i->number - % (t.tm_year + 1900) % (t.tm_mon + 1) % t.tm_mday - % t.tm_hour % t.tm_min % t.tm_sec - % (i->number == curGen ? "(current)" : ""); - } -} - - -static void deleteGeneration2(Globals & globals, unsigned int gen) -{ - if (globals.dryRun) - printMsg(lvlInfo, format("would remove generation %1%") % gen); - else { - printMsg(lvlInfo, format("removing generation %1%") % gen); - deleteGeneration(globals.profile, gen); - } - -} - - -static void opDeleteGenerations(Globals & globals, - Strings args, Strings opFlags, Strings opArgs) -{ - if (opFlags.size() > 0) - throw UsageError(format("unknown flag `%1%'") % opFlags.front()); - - PathLocks lock; - lockProfile(lock, globals.profile); - - int curGen; - Generations gens = findGenerations(globals.profile, curGen); - - for (Strings::iterator i = opArgs.begin(); i != opArgs.end(); ++i) { - - if (*i == "old") { - for (Generations::iterator j = gens.begin(); j != gens.end(); ++j) - if (j->number != curGen) - deleteGeneration2(globals, j->number); - } else if (i->size() >= 2 && tolower(*i->rbegin()) == 'd') { - time_t curTime = time(NULL); - time_t oldTime; - string strDays = string(*i, 0, i->size() - 1); - int days; - - if (!string2Int(strDays, days) || days < 1) - throw UsageError(format("invalid number of days specifier `%1%'") % *i); - - oldTime = curTime - days * 24 * 3600; - - for (Generations::iterator j = gens.begin(); j != gens.end(); ++j) { - if (j->number == curGen) continue; - - if (j->creationTime < oldTime) - deleteGeneration2(globals, j->number); - } - } else { - int n; - if (!string2Int(*i, n) || n < 0) - throw UsageError(format("invalid generation specifier `%1%'") % *i); - bool found = false; - for (Generations::iterator j = gens.begin(); j != gens.end(); ++j) { - if (j->number == n) { - deleteGeneration2(globals, j->number); - found = true; - break; - } - } - if (!found) - printMsg(lvlError, format("generation %1% does not exist") % n); - } - } -} - - -void run(Strings args) -{ - Strings opFlags, opArgs, remaining; - Operation op = 0; - - Globals globals; - - globals.instSource.type = srcUnknown; - globals.instSource.nixExprPath = getDefNixExprPath(); - globals.instSource.systemFilter = "*"; - - globals.dryRun = false; - globals.preserveInstalled = false; - globals.removeAll = false; - globals.prebuiltOnly = false; - - for (Strings::iterator i = args.begin(); i != args.end(); ) { - string arg = *i++; - - Operation oldOp = op; - - if (arg == "--install" || arg == "-i") - op = opInstall; - else if (parseOptionArg(arg, i, args.end(), - globals.state, globals.instSource.autoArgs)) - ; - else if (parseSearchPathArg(arg, i, args.end(), globals.state)) - ; - else if (arg == "--force-name") // undocumented flag for nix-install-package - globals.forceName = needArg(i, args, arg); - else if (arg == "--uninstall" || arg == "-e") - op = opUninstall; - else if (arg == "--upgrade" || arg == "-u") - op = opUpgrade; - else if (arg == "--set-flag") - op = opSetFlag; - else if (arg == "--set") - op = opSet; - else if (arg == "--query" || arg == "-q") - op = opQuery; - else if (arg == "--profile" || arg == "-p") - globals.profile = absPath(needArg(i, args, arg)); - else if (arg == "--file" || arg == "-f") - globals.instSource.nixExprPath = lookupFileArg(globals.state, needArg(i, args, arg)); - else if (arg == "--switch-profile" || arg == "-S") - op = opSwitchProfile; - else if (arg == "--switch-generation" || arg == "-G") - op = opSwitchGeneration; - else if (arg == "--rollback") - op = opRollback; - else if (arg == "--list-generations") - op = opListGenerations; - else if (arg == "--delete-generations") - op = opDeleteGenerations; - else if (arg == "--dry-run") { - printMsg(lvlInfo, "(dry run; not doing anything)"); - globals.dryRun = true; - } - else if (arg == "--system-filter") - globals.instSource.systemFilter = needArg(i, args, arg); - else if (arg == "--prebuilt-only" || arg == "-b") - globals.prebuiltOnly = true; - else if (arg == "--repair") - globals.state.repair = true; - else { - remaining.push_back(arg); - if (arg[0] == '-') { - opFlags.push_back(arg); - if (arg == "--from-profile") { /* !!! hack */ - if (i != args.end()) opFlags.push_back(*i++); - } - } else opArgs.push_back(arg); - } - - if (oldOp && oldOp != op) - throw UsageError("only one operation may be specified"); - } - - if (!op) throw UsageError("no operation specified"); - - if (globals.profile == "") - globals.profile = getEnv("NIX_PROFILE", ""); - - if (globals.profile == "") { - Path profileLink = getHomeDir() + "/.nix-profile"; - globals.profile = pathExists(profileLink) - ? absPath(readLink(profileLink), dirOf(profileLink)) - : canonPath(settings.nixStateDir + "/profiles/default"); - } - - store = openStore(); - - op(globals, remaining, opFlags, opArgs); - - globals.state.printStats(); -} - - -string programId = "nix-env"; diff --git a/src/nix-env/profiles.cc b/src/nix-env/profiles.cc deleted file mode 100644 index f7b306890d..0000000000 --- a/src/nix-env/profiles.cc +++ /dev/null @@ -1,147 +0,0 @@ -#include "profiles.hh" -#include "store-api.hh" -#include "util.hh" - -#include <sys/types.h> -#include <sys/stat.h> -#include <unistd.h> -#include <errno.h> -#include <stdio.h> - - -namespace nix { - - -static bool cmpGensByNumber(const Generation & a, const Generation & b) -{ - return a.number < b.number; -} - - -/* Parse a generation name of the format - `<profilename>-<number>-link'. */ -static int parseName(const string & profileName, const string & name) -{ - if (string(name, 0, profileName.size() + 1) != profileName + "-") return -1; - string s = string(name, profileName.size() + 1); - string::size_type p = s.find("-link"); - if (p == string::npos) return -1; - int n; - if (string2Int(string(s, 0, p), n) && n >= 0) - return n; - else - return -1; -} - - - -Generations findGenerations(Path profile, int & curGen) -{ - Generations gens; - - Path profileDir = dirOf(profile); - string profileName = baseNameOf(profile); - - Strings names = readDirectory(profileDir); - for (Strings::iterator i = names.begin(); i != names.end(); ++i) { - int n; - if ((n = parseName(profileName, *i)) != -1) { - Generation gen; - gen.path = profileDir + "/" + *i; - gen.number = n; - struct stat st; - if (lstat(gen.path.c_str(), &st) != 0) - throw SysError(format("statting `%1%'") % gen.path); - gen.creationTime = st.st_mtime; - gens.push_back(gen); - } - } - - gens.sort(cmpGensByNumber); - - curGen = pathExists(profile) - ? parseName(profileName, readLink(profile)) - : -1; - - return gens; -} - - -static void makeName(const Path & profile, unsigned int num, - Path & outLink) -{ - Path prefix = (format("%1%-%2%") % profile % num).str(); - outLink = prefix + "-link"; -} - - -Path createGeneration(Path profile, Path outPath) -{ - /* The new generation number should be higher than old the - previous ones. */ - int dummy; - Generations gens = findGenerations(profile, dummy); - unsigned int num = gens.size() > 0 ? gens.back().number : 0; - - /* Create the new generation. Note that addPermRoot() blocks if - the garbage collector is running to prevent the stuff we've - built from moving from the temporary roots (which the GC knows) - to the permanent roots (of which the GC would have a stale - view). If we didn't do it this way, the GC might remove the - user environment etc. we've just built. */ - Path generation; - makeName(profile, num + 1, generation); - addPermRoot(*store, outPath, generation, false, true); - - return generation; -} - - -static void removeFile(const Path & path) -{ - if (remove(path.c_str()) == -1) - throw SysError(format("cannot unlink `%1%'") % path); -} - - -void deleteGeneration(const Path & profile, unsigned int gen) -{ - Path generation; - makeName(profile, gen, generation); - removeFile(generation); -} - - -void switchLink(Path link, Path target) -{ - /* Hacky. */ - if (dirOf(target) == dirOf(link)) target = baseNameOf(target); - - Path tmp = canonPath(dirOf(link) + "/.new_" + baseNameOf(link)); - createSymlink(target, tmp); - /* The rename() system call is supposed to be essentially atomic - on Unix. That is, if we have links `current -> X' and - `new_current -> Y', and we rename new_current to current, a - process accessing current will see X or Y, but never a - file-not-found or other error condition. This is sufficient to - atomically switch user environments. */ - if (rename(tmp.c_str(), link.c_str()) != 0) - throw SysError(format("renaming `%1%' to `%2%'") % tmp % link); -} - - -void lockProfile(PathLocks & lock, const Path & profile) -{ - lock.lockPaths(singleton<PathSet>(profile), - (format("waiting for lock on profile `%1%'") % profile).str()); - lock.setDeletion(true); -} - - -string optimisticLockProfile(const Path & profile) -{ - return pathExists(profile) ? readLink(profile) : ""; -} - - -} diff --git a/src/nix-env/profiles.hh b/src/nix-env/profiles.hh deleted file mode 100644 index 30d2376d99..0000000000 --- a/src/nix-env/profiles.hh +++ /dev/null @@ -1,55 +0,0 @@ -#pragma once - -#include "types.hh" -#include "pathlocks.hh" - -#include <time.h> - - -namespace nix { - - -struct Generation -{ - int number; - Path path; - time_t creationTime; - Generation() - { - number = -1; - } - operator bool() const - { - return number != -1; - } -}; - -typedef list<Generation> Generations; - - -/* Returns the list of currently present generations for the specified - profile, sorted by generation number. */ -Generations findGenerations(Path profile, int & curGen); - -Path createGeneration(Path profile, Path outPath); - -void deleteGeneration(const Path & profile, unsigned int gen); - -void switchLink(Path link, Path target); - -/* Ensure exclusive access to a profile. Any command that modifies - the profile first acquires this lock. */ -void lockProfile(PathLocks & lock, const Path & profile); - -/* Optimistic locking is used by long-running operations like `nix-env - -i'. Instead of acquiring the exclusive lock for the entire - duration of the operation, we just perform the operation - optimistically (without an exclusive lock), and check at the end - whether the profile changed while we were busy (i.e., the symlink - target changed). If so, the operation is restarted. Restarting is - generally cheap, since the build results are still in the Nix - store. Most of the time, only the user environment has to be - rebuilt. */ -string optimisticLockProfile(const Path & profile); - -} diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc deleted file mode 100644 index a95c4cba93..0000000000 --- a/src/nix-env/user-env.cc +++ /dev/null @@ -1,149 +0,0 @@ -#include "user-env.hh" -#include "util.hh" -#include "derivations.hh" -#include "store-api.hh" -#include "globals.hh" -#include "shared.hh" -#include "eval.hh" -#include "eval-inline.hh" -#include "profiles.hh" - - -namespace nix { - - -DrvInfos queryInstalled(EvalState & state, const Path & userEnv) -{ - DrvInfos elems; - Path manifestFile = userEnv + "/manifest.nix"; - if (pathExists(manifestFile)) { - Value v; - state.evalFile(manifestFile, v); - Bindings bindings; - getDerivations(state, v, "", bindings, elems, false); - } - return elems; -} - - -bool createUserEnv(EvalState & state, DrvInfos & elems, - const Path & profile, bool keepDerivations, - const string & lockToken) -{ - /* Build the components in the user environment, if they don't - exist already. */ - PathSet drvsToBuild; - foreach (DrvInfos::iterator, i, elems) - if (i->queryDrvPath() != "") - drvsToBuild.insert(i->queryDrvPath()); - - debug(format("building user environment dependencies")); - store->buildPaths(drvsToBuild, state.repair ? bmRepair : bmNormal); - - /* Construct the whole top level derivation. */ - PathSet references; - Value manifest; - state.mkList(manifest, elems.size()); - unsigned int n = 0; - foreach (DrvInfos::iterator, i, elems) { - /* Create a pseudo-derivation containing the name, system, - output paths, and optionally the derivation path, as well - as the meta attributes. */ - Path drvPath = keepDerivations ? i->queryDrvPath() : ""; - - Value & v(*state.allocValue()); - manifest.list.elems[n++] = &v; - state.mkAttrs(v, 16); - - mkString(*state.allocAttr(v, state.sType), "derivation"); - mkString(*state.allocAttr(v, state.sName), i->name); - if (!i->system.empty()) - mkString(*state.allocAttr(v, state.sSystem), i->system); - mkString(*state.allocAttr(v, state.sOutPath), i->queryOutPath()); - if (drvPath != "") - mkString(*state.allocAttr(v, state.sDrvPath), i->queryDrvPath()); - - // Copy each output. - DrvInfo::Outputs outputs = i->queryOutputs(); - Value & vOutputs = *state.allocAttr(v, state.sOutputs); - state.mkList(vOutputs, outputs.size()); - unsigned int m = 0; - foreach (DrvInfo::Outputs::iterator, j, outputs) { - mkString(*(vOutputs.list.elems[m++] = state.allocValue()), j->first); - Value & vOutputs = *state.allocAttr(v, state.symbols.create(j->first)); - state.mkAttrs(vOutputs, 2); - mkString(*state.allocAttr(vOutputs, state.sOutPath), j->second); - - /* This is only necessary when installing store paths, e.g., - `nix-env -i /nix/store/abcd...-foo'. */ - store->addTempRoot(j->second); - store->ensurePath(j->second); - - references.insert(j->second); - } - - // Copy the meta attributes. - Value & vMeta = *state.allocAttr(v, state.sMeta); - state.mkAttrs(vMeta, 16); - StringSet metaNames = i->queryMetaNames(); - foreach (StringSet::iterator, j, metaNames) { - Value * v = i->queryMeta(*j); - if (!v) continue; - vMeta.attrs->push_back(Attr(state.symbols.create(*j), v)); - } - vMeta.attrs->sort(); - v.attrs->sort(); - - if (drvPath != "") references.insert(drvPath); - } - - /* Also write a copy of the list of user environment elements to - the store; we need it for future modifications of the - environment. */ - Path manifestFile = store->addTextToStore("env-manifest.nix", - (format("%1%") % manifest).str(), references); - - /* Get the environment builder expression. */ - Value envBuilder; - state.evalFile(state.findFile("nix/buildenv.nix"), envBuilder); - - /* Construct a Nix expression that calls the user environment - builder with the manifest as argument. */ - Value args, topLevel; - state.mkAttrs(args, 3); - mkString(*state.allocAttr(args, state.symbols.create("manifest")), - manifestFile, singleton<PathSet>(manifestFile)); - args.attrs->push_back(Attr(state.symbols.create("derivations"), &manifest)); - args.attrs->sort(); - mkApp(topLevel, envBuilder, args); - - /* Evaluate it. */ - debug("evaluating user environment builder"); - state.forceValue(topLevel); - PathSet context; - Path topLevelDrv = state.coerceToPath(*topLevel.attrs->find(state.sDrvPath)->value, context); - Path topLevelOut = state.coerceToPath(*topLevel.attrs->find(state.sOutPath)->value, context); - - /* Realise the resulting store expression. */ - debug("building user environment"); - store->buildPaths(singleton<PathSet>(topLevelDrv), state.repair ? bmRepair : bmNormal); - - /* Switch the current user environment to the output path. */ - PathLocks lock; - lockProfile(lock, profile); - - Path lockTokenCur = optimisticLockProfile(profile); - if (lockToken != lockTokenCur) { - printMsg(lvlError, format("profile `%1%' changed while we were busy; restarting") % profile); - return false; - } - - debug(format("switching to new user environment")); - Path generation = createGeneration(profile, topLevelOut); - switchLink(profile, generation); - - return true; -} - - -} diff --git a/src/nix-env/user-env.hh b/src/nix-env/user-env.hh deleted file mode 100644 index f188efe9b4..0000000000 --- a/src/nix-env/user-env.hh +++ /dev/null @@ -1,13 +0,0 @@ -#pragma once - -#include "get-drvs.hh" - -namespace nix { - -DrvInfos queryInstalled(EvalState & state, const Path & userEnv); - -bool createUserEnv(EvalState & state, DrvInfos & elems, - const Path & profile, bool keepDerivations, - const string & lockToken); - -} diff --git a/src/nix-hash/local.mk b/src/nix-hash/local.mk deleted file mode 100644 index 7c290ca846..0000000000 --- a/src/nix-hash/local.mk +++ /dev/null @@ -1,7 +0,0 @@ -programs += nix-hash - -nix-hash_DIR := $(d) - -nix-hash_SOURCES := $(d)/nix-hash.cc - -nix-hash_LIBS = libmain libstore libutil libformat diff --git a/src/nix-hash/nix-hash.cc b/src/nix-hash/nix-hash.cc deleted file mode 100644 index af3dda4ad7..0000000000 --- a/src/nix-hash/nix-hash.cc +++ /dev/null @@ -1,64 +0,0 @@ -#include "hash.hh" -#include "shared.hh" - -#include <iostream> - - -using namespace nix; - - -void printHelp() -{ - showManPage("nix-hash"); -} - - -void run(Strings args) -{ - HashType ht = htMD5; - bool flat = false; - bool base32 = false; - bool truncate = false; - enum { opHash, opTo32, opTo16 } op = opHash; - - Strings ss; - - for (Strings::iterator i = args.begin(); - i != args.end(); i++) - { - if (*i == "--flat") flat = true; - else if (*i == "--base32") base32 = true; - else if (*i == "--truncate") truncate = true; - else if (*i == "--type") { - ++i; - if (i == args.end()) throw UsageError("`--type' requires an argument"); - ht = parseHashType(*i); - if (ht == htUnknown) - throw UsageError(format("unknown hash type `%1%'") % *i); - } - else if (*i == "--to-base16") op = opTo16; - else if (*i == "--to-base32") op = opTo32; - else ss.push_back(*i); - } - - if (op == opHash) { - foreach (Strings::iterator, i, ss) { - Hash h = flat ? hashFile(ht, *i) : hashPath(ht, *i).first; - if (truncate && h.hashSize > 20) h = compressHash(h, 20); - std::cout << format("%1%\n") % - (base32 ? printHash32(h) : printHash(h)); - } - } - - else { - foreach (Strings::iterator, i, ss) { - Hash h = parseHash16or32(ht, *i); - std::cout << format("%1%\n") % - (op == opTo16 ? printHash(h) : printHash32(h)); - } - } -} - - -string programId = "nix-hash"; - diff --git a/src/nix-instantiate/local.mk b/src/nix-instantiate/local.mk deleted file mode 100644 index 7d1bc5ec9d..0000000000 --- a/src/nix-instantiate/local.mk +++ /dev/null @@ -1,7 +0,0 @@ -programs += nix-instantiate - -nix-instantiate_DIR := $(d) - -nix-instantiate_SOURCES := $(d)/nix-instantiate.cc - -nix-instantiate_LIBS = libexpr libmain libstore libutil libformat diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc deleted file mode 100644 index afd16b3a52..0000000000 --- a/src/nix-instantiate/nix-instantiate.cc +++ /dev/null @@ -1,185 +0,0 @@ -#include "globals.hh" -#include "shared.hh" -#include "eval.hh" -#include "eval-inline.hh" -#include "get-drvs.hh" -#include "attr-path.hh" -#include "value-to-xml.hh" -#include "util.hh" -#include "store-api.hh" -#include "common-opts.hh" -#include "misc.hh" - -#include <map> -#include <iostream> - - -using namespace nix; - - -void printHelp() -{ - showManPage("nix-instantiate"); -} - - -static Expr * parseStdin(EvalState & state) -{ - startNest(nest, lvlTalkative, format("parsing standard input")); - return state.parseExprFromString(drainFD(0), absPath(".")); -} - - -static Path gcRoot; -static int rootNr = 0; -static bool indirectRoot = false; - - -void processExpr(EvalState & state, const Strings & attrPaths, - bool parseOnly, bool strict, Bindings & autoArgs, - bool evalOnly, bool xmlOutput, bool location, Expr * e) -{ - if (parseOnly) { - std::cout << format("%1%\n") % *e; - return; - } - - Value vRoot; - state.eval(e, vRoot); - - foreach (Strings::const_iterator, i, attrPaths) { - Value & v(*findAlongAttrPath(state, *i, autoArgs, vRoot)); - state.forceValue(v); - - PathSet context; - if (evalOnly) - if (xmlOutput) - printValueAsXML(state, strict, location, v, std::cout, context); - else { - if (strict) state.strictForceValue(v); - std::cout << v << std::endl; - } - else { - DrvInfos drvs; - getDerivations(state, v, "", autoArgs, drvs, false); - foreach (DrvInfos::iterator, i, drvs) { - Path drvPath = i->queryDrvPath(); - - /* What output do we want? */ - string outputName = i->queryOutputName(); - if (outputName == "") - throw Error(format("derivation `%1%' lacks an `outputName' attribute ") % drvPath); - - if (gcRoot == "") - printGCWarning(); - else { - Path rootName = gcRoot; - if (++rootNr > 1) rootName += "-" + int2String(rootNr); - drvPath = addPermRoot(*store, drvPath, rootName, indirectRoot); - } - std::cout << format("%1%%2%\n") % drvPath % (outputName != "out" ? "!" + outputName : ""); - } - } - } -} - - -void run(Strings args) -{ - EvalState state; - Strings files; - bool readStdin = false; - bool fromArgs = false; - bool findFile = false; - bool evalOnly = false; - bool parseOnly = false; - bool xmlOutput = false; - bool xmlOutputSourceLocation = true; - bool strict = false; - Strings attrPaths; - Bindings autoArgs; - bool wantsReadWrite = false; - - for (Strings::iterator i = args.begin(); i != args.end(); ) { - string arg = *i++; - - if (arg == "-") - readStdin = true; - else if (arg == "--expr" || arg == "-E") - fromArgs = true; - else if (arg == "--eval" || arg == "--eval-only") - evalOnly = true; - else if (arg == "--read-write-mode") - wantsReadWrite = true; - else if (arg == "--parse" || arg == "--parse-only") - parseOnly = evalOnly = true; - else if (arg == "--find-file") - findFile = true; - else if (arg == "--attr" || arg == "-A") { - if (i == args.end()) - throw UsageError("`--attr' requires an argument"); - attrPaths.push_back(*i++); - } - else if (parseOptionArg(arg, i, args.end(), state, autoArgs)) - ; - else if (parseSearchPathArg(arg, i, args.end(), state)) - ; - else if (arg == "--add-root") { - if (i == args.end()) - throw UsageError("`--add-root' requires an argument"); - gcRoot = absPath(*i++); - } - else if (arg == "--indirect") - indirectRoot = true; - else if (arg == "--xml") - xmlOutput = true; - else if (arg == "--no-location") - xmlOutputSourceLocation = false; - else if (arg == "--strict") - strict = true; - else if (arg == "--repair") - state.repair = true; - else if (arg == "--dry-run") - settings.readOnlyMode = true; - else if (arg[0] == '-') - throw UsageError(format("unknown flag `%1%'") % arg); - else - files.push_back(arg); - } - - if (evalOnly && !wantsReadWrite) - settings.readOnlyMode = true; - - if (attrPaths.empty()) attrPaths.push_back(""); - - if (findFile) { - foreach (Strings::iterator, i, files) { - Path p = state.findFile(*i); - if (p == "") throw Error(format("unable to find `%1%'") % *i); - std::cout << p << std::endl; - } - return; - } - - store = openStore(); - - if (readStdin) { - Expr * e = parseStdin(state); - processExpr(state, attrPaths, parseOnly, strict, autoArgs, - evalOnly, xmlOutput, xmlOutputSourceLocation, e); - } else if (files.empty() && !fromArgs) - files.push_back("./default.nix"); - - foreach (Strings::iterator, i, files) { - Expr * e = fromArgs - ? state.parseExprFromString(*i, absPath(".")) - : state.parseExprFromFile(resolveExprPath(lookupFileArg(state, *i))); - processExpr(state, attrPaths, parseOnly, strict, autoArgs, - evalOnly, xmlOutput, xmlOutputSourceLocation, e); - } - - state.printStats(); -} - - -string programId = "nix-instantiate"; diff --git a/src/nix-log2xml/local.mk b/src/nix-log2xml/local.mk deleted file mode 100644 index 46eb2e02ca..0000000000 --- a/src/nix-log2xml/local.mk +++ /dev/null @@ -1,8 +0,0 @@ -programs += nix-log2xml - -nix-log2xml_DIR := $(d) - -nix-log2xml_SOURCES := $(d)/log2xml.cc - -$(foreach file, mark-errors.xsl log2html.xsl treebits.js, \ - $(eval $(call install-data-in, $(d)/$(file), $(datadir)/nix/log2html))) diff --git a/src/nix-log2xml/log2html.xsl b/src/nix-log2xml/log2html.xsl deleted file mode 100644 index 2093994706..0000000000 --- a/src/nix-log2xml/log2html.xsl +++ /dev/null @@ -1,83 +0,0 @@ -<?xml version="1.0"?> - -<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform"> - - <xsl:output method='html' encoding="UTF-8" - doctype-public="-//W3C//DTD HTML 4.01//EN" - doctype-system="http://www.w3.org/TR/html4/strict.dtd" /> - - <xsl:template match="logfile"> - <html> - <head> - <script type="text/javascript" src="treebits.js" /> - <link rel="stylesheet" href="logfile.css" type="text/css" /> - <title>Log File</title> - </head> - <body> - <ul class='toplevel'> - <xsl:for-each select='line|nest'> - <li> - <xsl:apply-templates select='.'/> - </li> - </xsl:for-each> - </ul> - </body> - </html> - </xsl:template> - - - <xsl:template match="nest"> - - <!-- The tree should be collapsed by default if all children are - unimportant or if the header is unimportant. --> -<!-- <xsl:variable name="collapsed" - select="count(.//line[not(@priority = 3)]) = 0 or ./head[@priority = 3]" /> --> - <xsl:variable name="collapsed" select="count(.//*[@error]) = 0"/> - - <xsl:variable name="style"><xsl:if test="$collapsed">display: none;</xsl:if></xsl:variable> - <xsl:variable name="arg"><xsl:choose><xsl:when test="$collapsed">true</xsl:when><xsl:otherwise>false</xsl:otherwise></xsl:choose></xsl:variable> - - <script type='text/javascript'>showTreeToggle(<xsl:value-of select="$collapsed"/>)</script> - <xsl:apply-templates select='head'/> - - <!-- Be careful to only generate <ul>s if there are <li>s, otherwise itâs malformed. --> - <xsl:if test="line|nest"> - - <ul class='nesting' style="{$style}"> - <xsl:for-each select='line|nest'> - - <!-- Is this the last line? If so, mark it as such so that it - can be rendered differently. --> - <xsl:variable name="class"><xsl:choose><xsl:when test="position() != last()">line</xsl:when><xsl:otherwise>lastline</xsl:otherwise></xsl:choose></xsl:variable> - - <li class='{$class}'> - <span class='lineconn' /> - <span class='linebody'> - <xsl:apply-templates select='.'/> - </span> - </li> - </xsl:for-each> - </ul> - </xsl:if> - - </xsl:template> - - - <xsl:template match="head|line"> - <code> - <xsl:if test="@error"> - <xsl:attribute name="class">error</xsl:attribute> - </xsl:if> - <xsl:apply-templates/> - </code> - </xsl:template> - - - <xsl:template match="storeref"> - <em class='storeref'> - <span class='popup'><xsl:apply-templates/></span> - <span class='elided'>/...</span><xsl:apply-templates select='name'/><xsl:apply-templates select='path'/> - </em> - </xsl:template> - -</xsl:stylesheet> \ No newline at end of file diff --git a/src/nix-log2xml/log2xml.cc b/src/nix-log2xml/log2xml.cc deleted file mode 100644 index 6645dc500f..0000000000 --- a/src/nix-log2xml/log2xml.cc +++ /dev/null @@ -1,200 +0,0 @@ -#include <vector> -#include <iostream> -#include <cstdio> -#include <string> -#include <cstring> - -using namespace std; - - -struct Decoder -{ - enum { stTop, stEscape, stCSI } state; - string line; - bool inHeader; - int level; - vector<int> args; - bool newNumber; - int priority; - bool ignoreLF; - int lineNo, charNo; - bool warning; - bool error; - - Decoder() - { - state = stTop; - line = ""; - inHeader = false; - level = 0; - priority = 1; - ignoreLF = false; - lineNo = 1; - charNo = 0; - warning = false; - error = false; - } - - void pushChar(char c); - - void finishLine(); - - void decodeFile(istream & st); -}; - - -void Decoder::pushChar(char c) -{ - if (c == '\n') { - lineNo++; - charNo = 0; - } else charNo++; - - switch (state) { - - case stTop: - if (c == '\e') { - state = stEscape; - } else if (c == '\n' && !ignoreLF) { - finishLine(); - } else line += c; - break; - - case stEscape: - if (c == '[') { - state = stCSI; - args.clear(); - newNumber = true; - } else - state = stTop; /* !!! wrong */ - break; - - case stCSI: - if (c >= 0x40 && c != 0x7e) { - state = stTop; - switch (c) { - case 'p': - if (line.size()) finishLine(); - level++; - inHeader = true; - cout << "<nest>" << endl; - priority = args.size() >= 1 ? args[0] : 1; - break; - case 'q': - if (line.size()) finishLine(); - if (level > 0) { - level--; - cout << "</nest>" << endl; - } else - cerr << "not enough nesting levels at line " - << lineNo << ", character " << charNo << endl; - break; - case 's': - if (line.size()) finishLine(); - priority = args.size() >= 1 ? args[0] : 1; - break; - case 'a': - ignoreLF = true; - break; - case 'b': - ignoreLF = false; - break; - case 'e': - error = true; - break; - case 'w': - warning = true; - break; - } - } else if (c >= '0' && c <= '9') { - int n = 0; - if (!newNumber) { - n = args.back() * 10; - args.pop_back(); - } - n += c - '0'; - args.push_back(n); - } - break; - - } -} - - -void Decoder::finishLine() -{ - string storeDir = "/nix/store/"; - int sz = storeDir.size(); - string tag = inHeader ? "head" : "line"; - cout << "<" << tag; - if (priority != 1) cout << " priority='" << priority << "'"; - if (warning) cout << " warning='1'"; - if (error) cout << " error='1'"; - cout << ">"; - - for (unsigned int i = 0; i < line.size(); i++) { - - if (line[i] == '<') cout << "<"; - else if (line[i] == '&') cout << "&"; - else if (line[i] == '\r') ; /* ignore carriage return */ - else if (line[i] < 32 && line[i] != 9) cout << "�"; - else if (i + sz + 33 < line.size() && - string(line, i, sz) == storeDir && - line[i + sz + 32] == '-') - { - int j = i + sz + 32; - /* skip name */ - while (!strchr("/\n\r\t ()[]:;?<>", line[j])) j++; - int k = j; - while (!strchr("\n\r\t ()[]:;?<>", line[k])) k++; - // !!! escaping - cout << "<storeref>" - << "<storedir>" - << string(line, i, sz) - << "</storedir>" - << "<hash>" - << string(line, i + sz, 32) - << "</hash>" - << "<name>" - << string(line, i + sz + 32, j - (i + sz + 32)) - << "</name>" - << "<path>" - << string(line, j, k - j) - << "</path>" - << "</storeref>"; - i = k - 1; - } else cout << line[i]; - } - - cout << "</" << tag << ">" << endl; - line = ""; - inHeader = false; - priority = 1; - warning = false; - error = false; -} - - -void Decoder::decodeFile(istream & st) -{ - int c; - - cout << "<logfile>" << endl; - - while ((c = st.get()) != EOF) { - pushChar(c); - } - - if (line.size()) finishLine(); - - while (level--) cout << "</nest>" << endl; - - cout << "</logfile>" << endl; -} - - -int main(int argc, char * * argv) -{ - Decoder dec; - dec.decodeFile(cin); -} diff --git a/src/nix-log2xml/logfile.css b/src/nix-log2xml/logfile.css deleted file mode 100644 index ed390d64a9..0000000000 --- a/src/nix-log2xml/logfile.css +++ /dev/null @@ -1,86 +0,0 @@ -body { - font-family: sans-serif; - background: white; -} - - -ul.nesting, ul.toplevel { - padding: 0; - margin: 0; -} - -ul.toplevel { - list-style-type: none; -} - -ul.nesting li.line, ul.nesting li.lastline { - position: relative; - list-style-type: none; -} - -ul.nesting li.line { - padding-left: 1.1em; -} - -ul.nesting li.lastline { - padding-left: 1.2em; // for the 0.1em border-left in .lastline > .lineconn -} - -li.line { - border-left: 0.1em solid #6185a0; -} - -li.line > span.lineconn, li.lastline > span.lineconn { - position: absolute; - height: 0.65em; - left: 0em; - width: 1em; - border-bottom: 0.1em solid #6185a0; -} - -li.lastline > span.lineconn { - border-left: 0.1em solid #6185a0; -} - - -em.storeref { - color: #500000; - position: relative; - width: 100%; -} - -em.storeref:hover { - background-color: #eeeeee; -} - -*.popup { - display: none; -/* background: url('http://losser.st-lab.cs.uu.nl/~mbravenb/menuback.png') repeat; */ - background: #ffffcd; - border: solid #555555 1px; - position: absolute; - top: 0em; - left: 0em; - margin: 0; - padding: 0; - z-index: 100; -} - -em.storeref:hover span.popup { - display: inline; -} - - -.toggle { - text-decoration: none; -} - -.showTree, .hideTree { - font-family: monospace; - font-size: larger; -} - -.error { - color: #ff0000; - font-weight: bold; -} \ No newline at end of file diff --git a/src/nix-log2xml/mark-errors.xsl b/src/nix-log2xml/mark-errors.xsl deleted file mode 100644 index 4e91913e57..0000000000 --- a/src/nix-log2xml/mark-errors.xsl +++ /dev/null @@ -1,24 +0,0 @@ -<?xml version="1.0"?> - -<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform"> - - <xsl:template match="@*|node()"> - <xsl:copy> - <xsl:apply-templates select="@*|node()"/> - </xsl:copy> - </xsl:template> - - <xsl:template match="line"> - <line> - <xsl:if test="contains(text(), ' *** ') or - contains(text(), 'LaTeX Error') or - contains(text(), 'FAIL:') or - contains(text(), ' error: ') or - true"> - <xsl:attribute name="error"></xsl:attribute> - </xsl:if> - <xsl:apply-templates select="@*|node()"/> - </line> - </xsl:template> - -</xsl:stylesheet> \ No newline at end of file diff --git a/src/nix-log2xml/treebits.js b/src/nix-log2xml/treebits.js deleted file mode 100644 index 3011b391d8..0000000000 --- a/src/nix-log2xml/treebits.js +++ /dev/null @@ -1,50 +0,0 @@ -/* Acknowledgement: this is based on the Wikipedia table-of-contents - * toggle. */ - - -var idCounter = 0; - - -function showTreeToggle(isHidden) { - if (document.getElementById) { - var id = "toggle_" + idCounter; - document.writeln( - '<a href="javascript:toggleTree(\'' + id + '\')" class="toggle" id="' + id + '">' + - '<span class="showTree" ' + (isHidden ? '' : 'style="display: none;"') + '>+</span>' + - '<span class="hideTree" ' + (isHidden ? 'style="display: none;"' : '') + '>-</span>' + - '</a>'); - idCounter = idCounter + 1; - } -} - - -function toggleTree(id) { - - var href = document.getElementById(id); - - var node = href; - var tree = null; - while (node != null) { - if (node.className == "nesting") tree = node; - node = node.nextSibling; - } - - node = href.firstChild; - var hideTree = null; - var showTree = null; - while (node != null) { - if (node.className == "showTree") showTree = node; - else if (node.className == "hideTree") hideTree = node; - node = node.nextSibling; - } - - if (tree.style.display == 'none') { - tree.style.display = ''; - hideTree.style.display = ''; - showTree.style.display = 'none'; - } else { - tree.style.display = 'none'; - hideTree.style.display = 'none'; - showTree.style.display = ''; - } -} diff --git a/src/nix-store/dotgraph.cc b/src/nix-store/dotgraph.cc deleted file mode 100644 index 72146eb68e..0000000000 --- a/src/nix-store/dotgraph.cc +++ /dev/null @@ -1,162 +0,0 @@ -#include "dotgraph.hh" -#include "util.hh" -#include "store-api.hh" - -#include <iostream> - - -using std::cout; - -namespace nix { - - -static string dotQuote(const string & s) -{ - return "\"" + s + "\""; -} - - -static string nextColour() -{ - static int n = 0; - static string colours[] = - { "black", "red", "green", "blue" - , "magenta", "burlywood" }; - return colours[n++ % (sizeof(colours) / sizeof(string))]; -} - - -static string makeEdge(const string & src, const string & dst) -{ - format f = format("%1% -> %2% [color = %3%];\n") - % dotQuote(src) % dotQuote(dst) % dotQuote(nextColour()); - return f.str(); -} - - -static string makeNode(const string & id, const string & label, - const string & colour) -{ - format f = format("%1% [label = %2%, shape = box, " - "style = filled, fillcolor = %3%];\n") - % dotQuote(id) % dotQuote(label) % dotQuote(colour); - return f.str(); -} - - -static string symbolicName(const string & path) -{ - string p = baseNameOf(path); - int dash = p.find('-'); - return string(p, dash + 1); -} - - -#if 0 -string pathLabel(const Path & nePath, const string & elemPath) -{ - return (string) nePath + "-" + elemPath; -} - - -void printClosure(const Path & nePath, const StoreExpr & fs) -{ - PathSet workList(fs.closure.roots); - PathSet doneSet; - - for (PathSet::iterator i = workList.begin(); i != workList.end(); ++i) { - cout << makeEdge(pathLabel(nePath, *i), nePath); - } - - while (!workList.empty()) { - Path path = *(workList.begin()); - workList.erase(path); - - if (doneSet.find(path) == doneSet.end()) { - doneSet.insert(path); - - ClosureElems::const_iterator elem = fs.closure.elems.find(path); - if (elem == fs.closure.elems.end()) - throw Error(format("bad closure, missing path `%1%'") % path); - - for (StringSet::const_iterator i = elem->second.refs.begin(); - i != elem->second.refs.end(); ++i) - { - workList.insert(*i); - cout << makeEdge(pathLabel(nePath, *i), pathLabel(nePath, path)); - } - - cout << makeNode(pathLabel(nePath, path), - symbolicName(path), "#ff0000"); - } - } -} -#endif - - -void printDotGraph(const PathSet & roots) -{ - PathSet workList(roots); - PathSet doneSet; - - cout << "digraph G {\n"; - - while (!workList.empty()) { - Path path = *(workList.begin()); - workList.erase(path); - - if (doneSet.find(path) != doneSet.end()) continue; - doneSet.insert(path); - - cout << makeNode(path, symbolicName(path), "#ff0000"); - - PathSet references; - store->queryReferences(path, references); - - for (PathSet::iterator i = references.begin(); - i != references.end(); ++i) - { - if (*i != path) { - workList.insert(*i); - cout << makeEdge(*i, path); - } - } - - -#if 0 - StoreExpr ne = storeExprFromPath(path); - - string label, colour; - - if (ne.type == StoreExpr::neDerivation) { - for (PathSet::iterator i = ne.derivation.inputs.begin(); - i != ne.derivation.inputs.end(); ++i) - { - workList.insert(*i); - cout << makeEdge(*i, path); - } - - label = "derivation"; - colour = "#00ff00"; - for (StringPairs::iterator i = ne.derivation.env.begin(); - i != ne.derivation.env.end(); ++i) - if (i->first == "name") label = i->second; - } - - else if (ne.type == StoreExpr::neClosure) { - label = "<closure>"; - colour = "#00ffff"; - printClosure(path, ne); - } - - else abort(); - - cout << makeNode(path, label, colour); -#endif - } - - cout << "}\n"; -} - - -} diff --git a/src/nix-store/dotgraph.hh b/src/nix-store/dotgraph.hh deleted file mode 100644 index 68410d8415..0000000000 --- a/src/nix-store/dotgraph.hh +++ /dev/null @@ -1,9 +0,0 @@ -#pragma once - -#include "types.hh" - -namespace nix { - -void printDotGraph(const PathSet & roots); - -} diff --git a/src/nix-store/local.mk b/src/nix-store/local.mk deleted file mode 100644 index 7f93e4c191..0000000000 --- a/src/nix-store/local.mk +++ /dev/null @@ -1,9 +0,0 @@ -programs += nix-store - -nix-store_DIR := $(d) - -nix-store_SOURCES := $(wildcard $(d)/*.cc) - -nix-store_LIBS = libmain libstore libutil libformat - -nix-store_LDFLAGS = -lbz2 diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc deleted file mode 100644 index 341a4f6776..0000000000 --- a/src/nix-store/nix-store.cc +++ /dev/null @@ -1,1007 +0,0 @@ -#include "globals.hh" -#include "misc.hh" -#include "archive.hh" -#include "shared.hh" -#include "dotgraph.hh" -#include "xmlgraph.hh" -#include "local-store.hh" -#include "util.hh" -#include "serve-protocol.hh" -#include "worker-protocol.hh" - -#include <iostream> -#include <algorithm> -#include <cstdio> - -#include <sys/types.h> -#include <sys/stat.h> -#include <fcntl.h> - -#include <bzlib.h> - - -using namespace nix; -using std::cin; -using std::cout; - - -typedef void (* Operation) (Strings opFlags, Strings opArgs); - - -void printHelp() -{ - showManPage("nix-store"); -} - - -static Path gcRoot; -static int rootNr = 0; -static bool indirectRoot = false; -static bool noOutput = false; - - -LocalStore & ensureLocalStore() -{ - LocalStore * store2(dynamic_cast<LocalStore *>(store.get())); - if (!store2) throw Error("you don't have sufficient rights to use this command"); - return *store2; -} - - -static Path useDeriver(Path path) -{ - if (!isDerivation(path)) { - path = store->queryDeriver(path); - if (path == "") - throw Error(format("deriver of path `%1%' is not known") % path); - } - return path; -} - - -/* Realise the given path. For a derivation that means build it; for - other paths it means ensure their validity. */ -static PathSet realisePath(Path path, bool build = true) -{ - DrvPathWithOutputs p = parseDrvPathWithOutputs(path); - - if (isDerivation(p.first)) { - if (build) store->buildPaths(singleton<PathSet>(path)); - Derivation drv = derivationFromPath(*store, p.first); - rootNr++; - - if (p.second.empty()) - foreach (DerivationOutputs::iterator, i, drv.outputs) p.second.insert(i->first); - - PathSet outputs; - foreach (StringSet::iterator, j, p.second) { - DerivationOutputs::iterator i = drv.outputs.find(*j); - if (i == drv.outputs.end()) - throw Error(format("derivation `%1%' does not have an output named `%2%'") % p.first % *j); - Path outPath = i->second.path; - if (gcRoot == "") - printGCWarning(); - else { - Path rootName = gcRoot; - if (rootNr > 1) rootName += "-" + int2String(rootNr); - if (i->first != "out") rootName += "-" + i->first; - outPath = addPermRoot(*store, outPath, rootName, indirectRoot); - } - outputs.insert(outPath); - } - return outputs; - } - - else { - if (build) store->ensurePath(path); - else if (!store->isValidPath(path)) throw Error(format("path `%1%' does not exist and cannot be created") % path); - if (gcRoot == "") - printGCWarning(); - else { - Path rootName = gcRoot; - rootNr++; - if (rootNr > 1) rootName += "-" + int2String(rootNr); - path = addPermRoot(*store, path, rootName, indirectRoot); - } - return singleton<PathSet>(path); - } -} - - -/* Realise the given paths. */ -static void opRealise(Strings opFlags, Strings opArgs) -{ - bool dryRun = false; - BuildMode buildMode = bmNormal; - bool ignoreUnknown = false; - - foreach (Strings::iterator, i, opFlags) - if (*i == "--dry-run") dryRun = true; - else if (*i == "--repair") buildMode = bmRepair; - else if (*i == "--check") buildMode = bmCheck; - else if (*i == "--ignore-unknown") ignoreUnknown = true; - else throw UsageError(format("unknown flag `%1%'") % *i); - - Paths paths; - foreach (Strings::iterator, i, opArgs) { - DrvPathWithOutputs p = parseDrvPathWithOutputs(*i); - paths.push_back(makeDrvPathWithOutputs(followLinksToStorePath(p.first), p.second)); - } - - unsigned long long downloadSize, narSize; - PathSet willBuild, willSubstitute, unknown; - queryMissing(*store, PathSet(paths.begin(), paths.end()), - willBuild, willSubstitute, unknown, downloadSize, narSize); - - if (ignoreUnknown) { - Paths paths2; - foreach (Paths::iterator, i, paths) - if (unknown.find(*i) == unknown.end()) paths2.push_back(*i); - paths = paths2; - unknown = PathSet(); - } - - printMissing(willBuild, willSubstitute, unknown, downloadSize, narSize); - - if (dryRun) return; - - /* Build all paths at the same time to exploit parallelism. */ - store->buildPaths(PathSet(paths.begin(), paths.end()), buildMode); - - if (!ignoreUnknown) - foreach (Paths::iterator, i, paths) { - PathSet paths = realisePath(*i, false); - if (!noOutput) - foreach (PathSet::iterator, j, paths) - cout << format("%1%\n") % *j; - } -} - - -/* Add files to the Nix store and print the resulting paths. */ -static void opAdd(Strings opFlags, Strings opArgs) -{ - if (!opFlags.empty()) throw UsageError("unknown flag"); - - for (Strings::iterator i = opArgs.begin(); i != opArgs.end(); ++i) - cout << format("%1%\n") % store->addToStore(*i); -} - - -/* Preload the output of a fixed-output derivation into the Nix - store. */ -static void opAddFixed(Strings opFlags, Strings opArgs) -{ - bool recursive = false; - - for (Strings::iterator i = opFlags.begin(); - i != opFlags.end(); ++i) - if (*i == "--recursive") recursive = true; - else throw UsageError(format("unknown flag `%1%'") % *i); - - if (opArgs.empty()) - throw UsageError("first argument must be hash algorithm"); - - HashType hashAlgo = parseHashType(opArgs.front()); - opArgs.pop_front(); - - for (Strings::iterator i = opArgs.begin(); i != opArgs.end(); ++i) - cout << format("%1%\n") % store->addToStore(*i, recursive, hashAlgo); -} - - -/* Hack to support caching in `nix-prefetch-url'. */ -static void opPrintFixedPath(Strings opFlags, Strings opArgs) -{ - bool recursive = false; - - for (Strings::iterator i = opFlags.begin(); - i != opFlags.end(); ++i) - if (*i == "--recursive") recursive = true; - else throw UsageError(format("unknown flag `%1%'") % *i); - - if (opArgs.size() != 3) - throw UsageError(format("`--print-fixed-path' requires three arguments")); - - Strings::iterator i = opArgs.begin(); - HashType hashAlgo = parseHashType(*i++); - string hash = *i++; - string name = *i++; - - cout << format("%1%\n") % - makeFixedOutputPath(recursive, hashAlgo, - parseHash16or32(hashAlgo, hash), name); -} - - -static PathSet maybeUseOutputs(const Path & storePath, bool useOutput, bool forceRealise) -{ - if (forceRealise) realisePath(storePath); - if (useOutput && isDerivation(storePath)) { - Derivation drv = derivationFromPath(*store, storePath); - PathSet outputs; - foreach (DerivationOutputs::iterator, i, drv.outputs) - outputs.insert(i->second.path); - return outputs; - } - else return singleton<PathSet>(storePath); -} - - -/* Some code to print a tree representation of a derivation dependency - graph. Topological sorting is used to keep the tree relatively - flat. */ - -const string treeConn = "+---"; -const string treeLine = "| "; -const string treeNull = " "; - - -static void printTree(const Path & path, - const string & firstPad, const string & tailPad, PathSet & done) -{ - if (done.find(path) != done.end()) { - cout << format("%1%%2% [...]\n") % firstPad % path; - return; - } - done.insert(path); - - cout << format("%1%%2%\n") % firstPad % path; - - PathSet references; - store->queryReferences(path, references); - - /* Topologically sort under the relation A < B iff A \in - closure(B). That is, if derivation A is an (possibly indirect) - input of B, then A is printed first. This has the effect of - flattening the tree, preventing deeply nested structures. */ - Paths sorted = topoSortPaths(*store, references); - reverse(sorted.begin(), sorted.end()); - - foreach (Paths::iterator, i, sorted) { - Paths::iterator j = i; ++j; - printTree(*i, tailPad + treeConn, - j == sorted.end() ? tailPad + treeNull : tailPad + treeLine, - done); - } -} - - -/* Perform various sorts of queries. */ -static void opQuery(Strings opFlags, Strings opArgs) -{ - enum { qOutputs, qRequisites, qReferences, qReferrers - , qReferrersClosure, qDeriver, qBinding, qHash, qSize - , qTree, qGraph, qXml, qResolve, qRoots } query = qOutputs; - bool useOutput = false; - bool includeOutputs = false; - bool forceRealise = false; - string bindingName; - - foreach (Strings::iterator, i, opFlags) - if (*i == "--outputs") query = qOutputs; - else if (*i == "--requisites" || *i == "-R") query = qRequisites; - else if (*i == "--references") query = qReferences; - else if (*i == "--referrers" || *i == "--referers") query = qReferrers; - else if (*i == "--referrers-closure" || *i == "--referers-closure") query = qReferrersClosure; - else if (*i == "--deriver" || *i == "-d") query = qDeriver; - else if (*i == "--binding" || *i == "-b") { - if (opArgs.size() == 0) - throw UsageError("expected binding name"); - bindingName = opArgs.front(); - opArgs.pop_front(); - query = qBinding; - } - else if (*i == "--hash") query = qHash; - else if (*i == "--size") query = qSize; - else if (*i == "--tree") query = qTree; - else if (*i == "--graph") query = qGraph; - else if (*i == "--xml") query = qXml; - else if (*i == "--resolve") query = qResolve; - else if (*i == "--roots") query = qRoots; - else if (*i == "--use-output" || *i == "-u") useOutput = true; - else if (*i == "--force-realise" || *i == "--force-realize" || *i == "-f") forceRealise = true; - else if (*i == "--include-outputs") includeOutputs = true; - else throw UsageError(format("unknown flag `%1%'") % *i); - - switch (query) { - - case qOutputs: { - foreach (Strings::iterator, i, opArgs) { - *i = followLinksToStorePath(*i); - if (forceRealise) realisePath(*i); - Derivation drv = derivationFromPath(*store, *i); - foreach (DerivationOutputs::iterator, j, drv.outputs) - cout << format("%1%\n") % j->second.path; - } - break; - } - - case qRequisites: - case qReferences: - case qReferrers: - case qReferrersClosure: { - PathSet paths; - foreach (Strings::iterator, i, opArgs) { - PathSet ps = maybeUseOutputs(followLinksToStorePath(*i), useOutput, forceRealise); - foreach (PathSet::iterator, j, ps) { - if (query == qRequisites) computeFSClosure(*store, *j, paths, false, includeOutputs); - else if (query == qReferences) store->queryReferences(*j, paths); - else if (query == qReferrers) store->queryReferrers(*j, paths); - else if (query == qReferrersClosure) computeFSClosure(*store, *j, paths, true); - } - } - Paths sorted = topoSortPaths(*store, paths); - for (Paths::reverse_iterator i = sorted.rbegin(); - i != sorted.rend(); ++i) - cout << format("%s\n") % *i; - break; - } - - case qDeriver: - foreach (Strings::iterator, i, opArgs) { - Path deriver = store->queryDeriver(followLinksToStorePath(*i)); - cout << format("%1%\n") % - (deriver == "" ? "unknown-deriver" : deriver); - } - break; - - case qBinding: - foreach (Strings::iterator, i, opArgs) { - Path path = useDeriver(followLinksToStorePath(*i)); - Derivation drv = derivationFromPath(*store, path); - StringPairs::iterator j = drv.env.find(bindingName); - if (j == drv.env.end()) - throw Error(format("derivation `%1%' has no environment binding named `%2%'") - % path % bindingName); - cout << format("%1%\n") % j->second; - } - break; - - case qHash: - case qSize: - foreach (Strings::iterator, i, opArgs) { - PathSet paths = maybeUseOutputs(followLinksToStorePath(*i), useOutput, forceRealise); - foreach (PathSet::iterator, j, paths) { - ValidPathInfo info = store->queryPathInfo(*j); - if (query == qHash) { - assert(info.hash.type == htSHA256); - cout << format("sha256:%1%\n") % printHash32(info.hash); - } else if (query == qSize) - cout << format("%1%\n") % info.narSize; - } - } - break; - - case qTree: { - PathSet done; - foreach (Strings::iterator, i, opArgs) - printTree(followLinksToStorePath(*i), "", "", done); - break; - } - - case qGraph: { - PathSet roots; - foreach (Strings::iterator, i, opArgs) { - PathSet paths = maybeUseOutputs(followLinksToStorePath(*i), useOutput, forceRealise); - roots.insert(paths.begin(), paths.end()); - } - printDotGraph(roots); - break; - } - - case qXml: { - PathSet roots; - foreach (Strings::iterator, i, opArgs) { - PathSet paths = maybeUseOutputs(followLinksToStorePath(*i), useOutput, forceRealise); - roots.insert(paths.begin(), paths.end()); - } - printXmlGraph(roots); - break; - } - - case qResolve: { - foreach (Strings::iterator, i, opArgs) - cout << format("%1%\n") % followLinksToStorePath(*i); - break; - } - - case qRoots: { - PathSet referrers; - foreach (Strings::iterator, i, opArgs) { - PathSet paths = maybeUseOutputs(followLinksToStorePath(*i), useOutput, forceRealise); - foreach (PathSet::iterator, j, paths) - computeFSClosure(*store, *j, referrers, true, - settings.gcKeepOutputs, settings.gcKeepDerivations); - } - Roots roots = store->findRoots(); - foreach (Roots::iterator, i, roots) - if (referrers.find(i->second) != referrers.end()) - cout << format("%1%\n") % i->first; - break; - } - - default: - abort(); - } -} - - -static string shellEscape(const string & s) -{ - string r; - foreach (string::const_iterator, i, s) - if (*i == '\'') r += "'\\''"; else r += *i; - return r; -} - - -static void opPrintEnv(Strings opFlags, Strings opArgs) -{ - if (!opFlags.empty()) throw UsageError("unknown flag"); - if (opArgs.size() != 1) throw UsageError("`--print-env' requires one derivation store path"); - - Path drvPath = opArgs.front(); - Derivation drv = derivationFromPath(*store, drvPath); - - /* Print each environment variable in the derivation in a format - that can be sourced by the shell. */ - foreach (StringPairs::iterator, i, drv.env) - cout << format("export %1%; %1%='%2%'\n") % i->first % shellEscape(i->second); - - /* Also output the arguments. This doesn't preserve whitespace in - arguments. */ - cout << "export _args; _args='"; - foreach (Strings::iterator, i, drv.args) { - if (i != drv.args.begin()) cout << ' '; - cout << shellEscape(*i); - } - cout << "'\n"; -} - - -static void opReadLog(Strings opFlags, Strings opArgs) -{ - if (!opFlags.empty()) throw UsageError("unknown flag"); - - foreach (Strings::iterator, i, opArgs) { - Path path = useDeriver(followLinksToStorePath(*i)); - - for (int j = 0; j <= 2; j++) { - if (j == 2) throw Error(format("build log of derivation `%1%' is not available") % path); - - string baseName = baseNameOf(path); - Path logPath = - j == 0 - ? (format("%1%/%2%/%3%/%4%") % settings.nixLogDir % drvsLogDir % string(baseName, 0, 2) % string(baseName, 2)).str() - : (format("%1%/%2%/%3%") % settings.nixLogDir % drvsLogDir % baseName).str(); - Path logBz2Path = logPath + ".bz2"; - - if (pathExists(logPath)) { - /* !!! Make this run in O(1) memory. */ - string log = readFile(logPath); - writeFull(STDOUT_FILENO, (const unsigned char *) log.data(), log.size()); - break; - } - - else if (pathExists(logBz2Path)) { - AutoCloseFD fd = open(logBz2Path.c_str(), O_RDONLY); - FILE * f = 0; - if (fd == -1 || (f = fdopen(fd.borrow(), "r")) == 0) - throw SysError(format("opening file `%1%'") % logBz2Path); - int err; - BZFILE * bz = BZ2_bzReadOpen(&err, f, 0, 0, 0, 0); - if (!bz) throw Error(format("cannot open bzip2 file `%1%'") % logBz2Path); - unsigned char buf[128 * 1024]; - do { - int n = BZ2_bzRead(&err, bz, buf, sizeof(buf)); - if (err != BZ_OK && err != BZ_STREAM_END) - throw Error(format("error reading bzip2 file `%1%'") % logBz2Path); - writeFull(STDOUT_FILENO, buf, n); - } while (err != BZ_STREAM_END); - BZ2_bzReadClose(&err, bz); - break; - } - } - } -} - - -static void opDumpDB(Strings opFlags, Strings opArgs) -{ - if (!opFlags.empty()) throw UsageError("unknown flag"); - if (!opArgs.empty()) - throw UsageError("no arguments expected"); - PathSet validPaths = store->queryAllValidPaths(); - foreach (PathSet::iterator, i, validPaths) - cout << store->makeValidityRegistration(singleton<PathSet>(*i), true, true); -} - - -static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) -{ - ValidPathInfos infos; - - while (1) { - ValidPathInfo info = decodeValidPathInfo(cin, hashGiven); - if (info.path == "") break; - if (!store->isValidPath(info.path) || reregister) { - /* !!! races */ - if (canonicalise) - canonicalisePathMetaData(info.path, -1); - if (!hashGiven) { - HashResult hash = hashPath(htSHA256, info.path); - info.hash = hash.first; - info.narSize = hash.second; - } - infos.push_back(info); - } - } - - ensureLocalStore().registerValidPaths(infos); -} - - -static void opLoadDB(Strings opFlags, Strings opArgs) -{ - if (!opFlags.empty()) throw UsageError("unknown flag"); - if (!opArgs.empty()) - throw UsageError("no arguments expected"); - registerValidity(true, true, false); -} - - -static void opRegisterValidity(Strings opFlags, Strings opArgs) -{ - bool reregister = false; // !!! maybe this should be the default - bool hashGiven = false; - - for (Strings::iterator i = opFlags.begin(); - i != opFlags.end(); ++i) - if (*i == "--reregister") reregister = true; - else if (*i == "--hash-given") hashGiven = true; - else throw UsageError(format("unknown flag `%1%'") % *i); - - if (!opArgs.empty()) throw UsageError("no arguments expected"); - - registerValidity(reregister, hashGiven, true); -} - - -static void opCheckValidity(Strings opFlags, Strings opArgs) -{ - bool printInvalid = false; - - for (Strings::iterator i = opFlags.begin(); - i != opFlags.end(); ++i) - if (*i == "--print-invalid") printInvalid = true; - else throw UsageError(format("unknown flag `%1%'") % *i); - - for (Strings::iterator i = opArgs.begin(); - i != opArgs.end(); ++i) - { - Path path = followLinksToStorePath(*i); - if (!store->isValidPath(path)) { - if (printInvalid) - cout << format("%1%\n") % path; - else - throw Error(format("path `%1%' is not valid") % path); - } - } -} - - -static string showBytes(unsigned long long bytes) -{ - return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str(); -} - - -struct PrintFreed -{ - bool show; - const GCResults & results; - PrintFreed(bool show, const GCResults & results) - : show(show), results(results) { } - ~PrintFreed() - { - if (show) - cout << format("%1% store paths deleted, %2% freed\n") - % results.paths.size() - % showBytes(results.bytesFreed); - } -}; - - -static void opGC(Strings opFlags, Strings opArgs) -{ - bool printRoots = false; - GCOptions options; - options.action = GCOptions::gcDeleteDead; - - GCResults results; - - /* Do what? */ - foreach (Strings::iterator, i, opFlags) - if (*i == "--print-roots") printRoots = true; - else if (*i == "--print-live") options.action = GCOptions::gcReturnLive; - else if (*i == "--print-dead") options.action = GCOptions::gcReturnDead; - else if (*i == "--delete") options.action = GCOptions::gcDeleteDead; - else if (*i == "--max-freed") { - long long maxFreed = getIntArg<long long>(*i, i, opFlags.end(), true); - options.maxFreed = maxFreed >= 0 ? maxFreed : 0; - } - else throw UsageError(format("bad sub-operation `%1%' in GC") % *i); - - if (!opArgs.empty()) throw UsageError("no arguments expected"); - - if (printRoots) { - Roots roots = store->findRoots(); - foreach (Roots::iterator, i, roots) - cout << i->first << " -> " << i->second << std::endl; - } - - else { - PrintFreed freed(options.action == GCOptions::gcDeleteDead, results); - store->collectGarbage(options, results); - - if (options.action != GCOptions::gcDeleteDead) - foreach (PathSet::iterator, i, results.paths) - cout << *i << std::endl; - } -} - - -/* Remove paths from the Nix store if possible (i.e., if they do not - have any remaining referrers and are not reachable from any GC - roots). */ -static void opDelete(Strings opFlags, Strings opArgs) -{ - GCOptions options; - options.action = GCOptions::gcDeleteSpecific; - - foreach (Strings::iterator, i, opFlags) - if (*i == "--ignore-liveness") options.ignoreLiveness = true; - else throw UsageError(format("unknown flag `%1%'") % *i); - - foreach (Strings::iterator, i, opArgs) - options.pathsToDelete.insert(followLinksToStorePath(*i)); - - GCResults results; - PrintFreed freed(true, results); - store->collectGarbage(options, results); -} - - -/* Dump a path as a Nix archive. The archive is written to standard - output. */ -static void opDump(Strings opFlags, Strings opArgs) -{ - if (!opFlags.empty()) throw UsageError("unknown flag"); - if (opArgs.size() != 1) throw UsageError("only one argument allowed"); - - FdSink sink(STDOUT_FILENO); - string path = *opArgs.begin(); - dumpPath(path, sink); -} - - -/* Restore a value from a Nix archive. The archive is read from - standard input. */ -static void opRestore(Strings opFlags, Strings opArgs) -{ - if (!opFlags.empty()) throw UsageError("unknown flag"); - if (opArgs.size() != 1) throw UsageError("only one argument allowed"); - - FdSource source(STDIN_FILENO); - restorePath(*opArgs.begin(), source); -} - - -static void opExport(Strings opFlags, Strings opArgs) -{ - bool sign = false; - for (Strings::iterator i = opFlags.begin(); - i != opFlags.end(); ++i) - if (*i == "--sign") sign = true; - else throw UsageError(format("unknown flag `%1%'") % *i); - - FdSink sink(STDOUT_FILENO); - Paths sorted = topoSortPaths(*store, PathSet(opArgs.begin(), opArgs.end())); - reverse(sorted.begin(), sorted.end()); - exportPaths(*store, sorted, sign, sink); -} - - -static void opImport(Strings opFlags, Strings opArgs) -{ - bool requireSignature = false; - foreach (Strings::iterator, i, opFlags) - if (*i == "--require-signature") requireSignature = true; - else throw UsageError(format("unknown flag `%1%'") % *i); - - if (!opArgs.empty()) throw UsageError("no arguments expected"); - - FdSource source(STDIN_FILENO); - Paths paths = store->importPaths(requireSignature, source); - - foreach (Paths::iterator, i, paths) - cout << format("%1%\n") % *i << std::flush; -} - - -/* Initialise the Nix databases. */ -static void opInit(Strings opFlags, Strings opArgs) -{ - if (!opFlags.empty()) throw UsageError("unknown flag"); - if (!opArgs.empty()) - throw UsageError("no arguments expected"); - /* Doesn't do anything right now; database tables are initialised - automatically. */ -} - - -/* Verify the consistency of the Nix environment. */ -static void opVerify(Strings opFlags, Strings opArgs) -{ - if (!opArgs.empty()) - throw UsageError("no arguments expected"); - - bool checkContents = false; - bool repair = false; - - for (Strings::iterator i = opFlags.begin(); - i != opFlags.end(); ++i) - if (*i == "--check-contents") checkContents = true; - else if (*i == "--repair") repair = true; - else throw UsageError(format("unknown flag `%1%'") % *i); - - if (ensureLocalStore().verifyStore(checkContents, repair)) { - printMsg(lvlError, "warning: not all errors were fixed"); - exitCode = 1; - } -} - - -/* Verify whether the contents of the given store path have not changed. */ -static void opVerifyPath(Strings opFlags, Strings opArgs) -{ - if (!opFlags.empty()) - throw UsageError("no flags expected"); - - foreach (Strings::iterator, i, opArgs) { - Path path = followLinksToStorePath(*i); - printMsg(lvlTalkative, format("checking path `%1%'...") % path); - ValidPathInfo info = store->queryPathInfo(path); - HashResult current = hashPath(info.hash.type, path); - if (current.first != info.hash) { - printMsg(lvlError, - format("path `%1%' was modified! expected hash `%2%', got `%3%'") - % path % printHash(info.hash) % printHash(current.first)); - exitCode = 1; - } - } -} - - -/* Repair the contents of the given path by redownloading it using a - substituter (if available). */ -static void opRepairPath(Strings opFlags, Strings opArgs) -{ - if (!opFlags.empty()) - throw UsageError("no flags expected"); - - foreach (Strings::iterator, i, opArgs) { - Path path = followLinksToStorePath(*i); - ensureLocalStore().repairPath(path); - } -} - - -static void showOptimiseStats(OptimiseStats & stats) -{ - printMsg(lvlError, - format("%1% freed by hard-linking %2% files; there are %3% files with equal contents out of %4% files in total") - % showBytes(stats.bytesFreed) - % stats.filesLinked - % stats.sameContents - % stats.totalFiles); -} - - -/* Optimise the disk space usage of the Nix store by hard-linking - files with the same contents. */ -static void opOptimise(Strings opFlags, Strings opArgs) -{ - if (!opArgs.empty() || !opFlags.empty()) - throw UsageError("no arguments expected"); - - OptimiseStats stats; - try { - ensureLocalStore().optimiseStore(stats); - } catch (...) { - showOptimiseStats(stats); - throw; - } - showOptimiseStats(stats); -} - - -static void opQueryFailedPaths(Strings opFlags, Strings opArgs) -{ - if (!opArgs.empty() || !opFlags.empty()) - throw UsageError("no arguments expected"); - PathSet failed = store->queryFailedPaths(); - foreach (PathSet::iterator, i, failed) - cout << format("%1%\n") % *i; -} - - -static void opClearFailedPaths(Strings opFlags, Strings opArgs) -{ - if (!opFlags.empty()) - throw UsageError("no flags expected"); - store->clearFailedPaths(PathSet(opArgs.begin(), opArgs.end())); -} - - -/* Serve the nix store in a way usable by a restricted ssh user. */ -static void opServe(Strings opFlags, Strings opArgs) -{ - if (!opArgs.empty() || !opFlags.empty()) - throw UsageError("no arguments or flags expected"); - - FdSource in(STDIN_FILENO); - FdSink out(STDOUT_FILENO); - - /* Exchange the greeting. */ - unsigned int magic = readInt(in); - if (magic != SERVE_MAGIC_1) throw Error("protocol mismatch"); - writeInt(SERVE_MAGIC_2, out); - writeInt(SERVE_PROTOCOL_VERSION, out); - out.flush(); - readInt(in); // Client version, unused for now - - ServeCommand cmd = (ServeCommand) readInt(in); - switch (cmd) { - case cmdQuery: - while (true) { - QueryCommand qCmd; - try { - qCmd = (QueryCommand) readInt(in); - } catch (EndOfFile & e) { - break; - } - switch (qCmd) { - case qCmdHave: { - PathSet paths = readStorePaths<PathSet>(in); - writeStrings(store->queryValidPaths(paths), out); - break; - } - case qCmdInfo: { - PathSet paths = readStorePaths<PathSet>(in); - // !!! Maybe we want a queryPathInfos? - foreach (PathSet::iterator, i, paths) { - if (!store->isValidPath(*i)) - continue; - ValidPathInfo info = store->queryPathInfo(*i); - writeString(info.path, out); - writeString(info.deriver, out); - writeStrings(info.references, out); - // !!! Maybe we want compression? - writeLongLong(info.narSize, out); // downloadSize - writeLongLong(info.narSize, out); - } - writeString("", out); - break; - } - default: - throw Error(format("unknown serve query `%1%'") % cmd); - } - out.flush(); - } - break; - case cmdSubstitute: - dumpPath(readString(in), out); - break; - default: - throw Error(format("unknown serve command `%1%'") % cmd); - } -} - - -/* Scan the arguments; find the operation, set global flags, put all - other flags in a list, and put all other arguments in another - list. */ -void run(Strings args) -{ - Strings opFlags, opArgs; - Operation op = 0; - - for (Strings::iterator i = args.begin(); i != args.end(); ) { - string arg = *i++; - - Operation oldOp = op; - - if (arg == "--realise" || arg == "--realize" || arg == "-r") - op = opRealise; - else if (arg == "--add" || arg == "-A") - op = opAdd; - else if (arg == "--add-fixed") - op = opAddFixed; - else if (arg == "--print-fixed-path") - op = opPrintFixedPath; - else if (arg == "--delete") - op = opDelete; - else if (arg == "--query" || arg == "-q") - op = opQuery; - else if (arg == "--print-env") - op = opPrintEnv; - else if (arg == "--read-log" || arg == "-l") - op = opReadLog; - else if (arg == "--dump-db") - op = opDumpDB; - else if (arg == "--load-db") - op = opLoadDB; - else if (arg == "--register-validity") - op = opRegisterValidity; - else if (arg == "--check-validity") - op = opCheckValidity; - else if (arg == "--gc") - op = opGC; - else if (arg == "--dump") - op = opDump; - else if (arg == "--restore") - op = opRestore; - else if (arg == "--export") - op = opExport; - else if (arg == "--import") - op = opImport; - else if (arg == "--init") - op = opInit; - else if (arg == "--verify") - op = opVerify; - else if (arg == "--verify-path") - op = opVerifyPath; - else if (arg == "--repair-path") - op = opRepairPath; - else if (arg == "--optimise" || arg == "--optimize") - op = opOptimise; - else if (arg == "--query-failed-paths") - op = opQueryFailedPaths; - else if (arg == "--clear-failed-paths") - op = opClearFailedPaths; - else if (arg == "--add-root") { - if (i == args.end()) - throw UsageError("`--add-root requires an argument"); - gcRoot = absPath(*i++); - } - else if (arg == "--indirect") - indirectRoot = true; - else if (arg == "--no-output") - noOutput = true; - else if (arg == "--serve") - op = opServe; - else if (arg[0] == '-') { - opFlags.push_back(arg); - if (arg == "--max-freed" || arg == "--max-links" || arg == "--max-atime") { /* !!! hack */ - if (i != args.end()) opFlags.push_back(*i++); - } - } - else - opArgs.push_back(arg); - - if (oldOp && oldOp != op) - throw UsageError("only one operation may be specified"); - } - - if (!op) throw UsageError("no operation specified"); - - if (op != opDump && op != opRestore) /* !!! hack */ - store = openStore(op != opGC); - - op(opFlags, opArgs); -} - - -string programId = "nix-store"; diff --git a/src/nix-store/serve-protocol.hh b/src/nix-store/serve-protocol.hh deleted file mode 100644 index 69277bc1b9..0000000000 --- a/src/nix-store/serve-protocol.hh +++ /dev/null @@ -1,24 +0,0 @@ -#pragma once - -namespace nix { - - -#define SERVE_MAGIC_1 0x390c9deb -#define SERVE_MAGIC_2 0x5452eecb - -#define SERVE_PROTOCOL_VERSION 0x101 -#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) -#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) - - -typedef enum { - cmdQuery = 0, - cmdSubstitute = 1, -} ServeCommand; - -typedef enum { - qCmdHave = 0, - qCmdInfo = 1, -} QueryCommand; - -} diff --git a/src/nix-store/xmlgraph.cc b/src/nix-store/xmlgraph.cc deleted file mode 100644 index 1b3ad3d28a..0000000000 --- a/src/nix-store/xmlgraph.cc +++ /dev/null @@ -1,71 +0,0 @@ -#include "xmlgraph.hh" -#include "util.hh" -#include "store-api.hh" - -#include <iostream> - - -using std::cout; - -namespace nix { - - -static inline const string & xmlQuote(const string & s) -{ - // Luckily, store paths shouldn't contain any character that needs to be - // quoted. - return s; -} - - -static string makeEdge(const string & src, const string & dst) -{ - format f = format(" <edge src=\"%1%\" dst=\"%2%\"/>\n") - % xmlQuote(src) % xmlQuote(dst); - return f.str(); -} - - -static string makeNode(const string & id) -{ - format f = format(" <node name=\"%1%\"/>\n") % xmlQuote(id); - return f.str(); -} - - -void printXmlGraph(const PathSet & roots) -{ - PathSet workList(roots); - PathSet doneSet; - - cout << "<?xml version='1.0' encoding='utf-8'?>\n" - << "<nix>\n"; - - while (!workList.empty()) { - Path path = *(workList.begin()); - workList.erase(path); - - if (doneSet.find(path) != doneSet.end()) continue; - doneSet.insert(path); - - cout << makeNode(path); - - PathSet references; - store->queryReferences(path, references); - - for (PathSet::iterator i = references.begin(); - i != references.end(); ++i) - { - if (*i != path) { - workList.insert(*i); - cout << makeEdge(*i, path); - } - } - - } - - cout << "</nix>\n"; -} - - -} diff --git a/src/nix-store/xmlgraph.hh b/src/nix-store/xmlgraph.hh deleted file mode 100644 index c2216c5a46..0000000000 --- a/src/nix-store/xmlgraph.hh +++ /dev/null @@ -1,9 +0,0 @@ -#pragma once - -#include "types.hh" - -namespace nix { - -void printXmlGraph(const PathSet & roots); - -} |