From 44057a461b1fa8102938c4e0f54d7cbc9dd09b03 Mon Sep 17 00:00:00 2001
From: Ricardo Wurmus <rekado@elephly.net>
Date: Wed, 30 May 2018 09:55:28 +0200
Subject: tests: Fix arguments in pack test.
This is a follow-up to commit 5ffac538aa604b71814ac74579626f0d3110b96e.
* tests/pack.scm (self-contained-tarball): Adjust arguments to
"self-contained-tarball".
---
tests/pack.scm | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
(limited to 'tests')
diff --git a/tests/pack.scm b/tests/pack.scm
index 3bce715075..fcc53d12ef 100644
--- a/tests/pack.scm
+++ b/tests/pack.scm
@@ -1,5 +1,6 @@
;;; GNU Guix --- Functional package management for GNU
;;; Copyright © 2017 Ludovic Courtès <ludo@gnu.org>
+;;; Copyright © 2018 Ricardo Wurmus <rekado@elephly.net>
;;;
;;; This file is part of GNU Guix.
;;;
@@ -61,7 +62,7 @@
#:symlinks '(("/bin/Guile"
-> "bin/guile"))
#:compressor %gzip-compressor
- #:tar %tar-bootstrap))
+ #:archiver %tar-bootstrap))
(check (gexp->derivation
"check-tarball"
#~(let ((guile (string-append "." #$profile "/bin")))
--
cgit 1.4.1
From ccc951cab3172adfdaf6fd2dfa8f8cdb98358a69 Mon Sep 17 00:00:00 2001
From: Ludovic Courtès <ludo@gnu.org>
Date: Fri, 1 Jun 2018 15:17:41 +0200
Subject: pack: Adjust test to expect relative symlinks.
Reported by Chris Marusich <cmmarusich@gmail.com>.
Fixes <https://bugs.gnu.org/31560>.
* tests/pack.scm ("self-contained-tarball"): Rename 'guile' to 'bin'.
Expect 'bin/Guile' to be a relative symlink.
---
tests/pack.scm | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
(limited to 'tests')
diff --git a/tests/pack.scm b/tests/pack.scm
index fcc53d12ef..d4596f863a 100644
--- a/tests/pack.scm
+++ b/tests/pack.scm
@@ -1,5 +1,5 @@
;;; GNU Guix --- Functional package management for GNU
-;;; Copyright © 2017 Ludovic Courtès <ludo@gnu.org>
+;;; Copyright © 2017, 2018 Ludovic Courtès <ludo@gnu.org>
;;; Copyright © 2018 Ricardo Wurmus <rekado@elephly.net>
;;;
;;; This file is part of GNU Guix.
@@ -65,17 +65,17 @@
#:archiver %tar-bootstrap))
(check (gexp->derivation
"check-tarball"
- #~(let ((guile (string-append "." #$profile "/bin")))
+ #~(let ((bin (string-append "." #$profile "/bin")))
(setenv "PATH"
(string-append #$%tar-bootstrap "/bin"))
(system* "tar" "xvf" #$tarball)
(mkdir #$output)
(exit
- (and (file-exists? (string-append guile "/guile"))
+ (and (file-exists? (string-append bin "/guile"))
(string=? (string-append #$%bootstrap-guile "/bin")
- (readlink guile))
- (string=? (string-append (string-drop guile 1)
- "/guile")
+ (readlink bin))
+ (string=? (string-append ".." #$profile
+ "/bin/guile")
(readlink "bin/Guile"))))))))
(built-derivations (list check))))
--
cgit 1.4.1
From 838e17d8050236a6d3ffde991fb0035412eb3046 Mon Sep 17 00:00:00 2001
From: Ludovic Courtès <ludo@gnu.org>
Date: Mon, 28 May 2018 18:14:37 +0200
Subject: gexp: Add 'with-extensions'.
* guix/gexp.scm (<gexp>)[extensions]: New field.
(gexp-attribute): New procedure.
(gexp-modules): Write in terms of 'gexp-attribute'.
(gexp-extensions): New procedure.
(gexp->derivation): Add #:effective-version.
[extension-flags]: New procedure.
Honor extensions of EXP.
(current-imported-extensions): New syntax parameter.
(with-extensions): New macro.
(gexp): Honor CURRENT-IMPORTED-EXTENSIONS.
(compiled-modules): Add #:extensions and honor it.
(load-path-expression): Likewise.
(gexp->script, gexp->file): Honor extensions.
* tests/gexp.scm (%extension-package): New variable.
("gexp-extensions & ungexp")
("gexp-extensions & ungexp-splicing")
("gexp-extensions and literal Scheme object")
("gexp->derivation & with-extensions")
("program-file & with-extensions"): New tests.
* doc/guix.texi (G-Expressions): Document 'with-extensions'.
---
.dir-locals.el | 1 +
doc/guix.texi | 33 ++++++++++++
guix/gexp.scm | 168 ++++++++++++++++++++++++++++++++++++++++++---------------
tests/gexp.scm | 86 +++++++++++++++++++++++++++++
4 files changed, 246 insertions(+), 42 deletions(-)
(limited to 'tests')
diff --git a/.dir-locals.el b/.dir-locals.el
index dac6cb1453..2db751ca22 100644
--- a/.dir-locals.el
+++ b/.dir-locals.el
@@ -73,6 +73,7 @@
(eval . (put 'run-with-state 'scheme-indent-function 1))
(eval . (put 'wrap-program 'scheme-indent-function 1))
(eval . (put 'with-imported-modules 'scheme-indent-function 1))
+ (eval . (put 'with-extensions 'scheme-indent-function 1))
(eval . (put 'call-with-container 'scheme-indent-function 1))
(eval . (put 'container-excursion 'scheme-indent-function 1))
diff --git a/doc/guix.texi b/doc/guix.texi
index 3b5078741d..77bdaa50eb 100644
--- a/doc/guix.texi
+++ b/doc/guix.texi
@@ -5064,6 +5064,23 @@ headers, which comes in handy in this case:
@dots{})))
@end example
+@cindex extensions, for gexps
+@findex with-extensions
+In the same vein, sometimes you want to import not just pure-Scheme
+modules, but also ``extensions'' such as Guile bindings to C libraries
+or other ``full-blown'' packages. Say you need the @code{guile-json}
+package available on the build side, here's how you would do it:
+
+@example
+(use-modules (gnu packages guile)) ;for 'guile-json'
+
+(with-extensions (list guile-json)
+ (gexp->derivation "something-with-json"
+ #~(begin
+ (use-modules (json))
+ @dots{})))
+@end example
+
The syntactic form to construct gexps is summarized below.
@deffn {Scheme Syntax} #~@var{exp}
@@ -5147,6 +5164,18 @@ directly defined in @var{body}@dots{}, but not on those defined, say, in
procedures called from @var{body}@dots{}.
@end deffn
+@deffn {Scheme Syntax} with-extensions @var{extensions} @var{body}@dots{}
+Mark the gexps defined in @var{body}@dots{} as requiring
+@var{extensions} in their build and execution environment.
+@var{extensions} is typically a list of package objects such as those
+defined in the @code{(gnu packages guile)} module.
+
+Concretely, the packages listed in @var{extensions} are added to the
+load path while compiling imported modules in @var{body}@dots{}; they
+are also added to the load path of the gexp returned by
+@var{body}@dots{}.
+@end deffn
+
@deffn {Scheme Procedure} gexp? @var{obj}
Return @code{#t} if @var{obj} is a G-expression.
@end deffn
@@ -5161,6 +5190,7 @@ information about monads.)
[#:hash #f] [#:hash-algo #f] @
[#:recursive? #f] [#:env-vars '()] [#:modules '()] @
[#:module-path @var{%load-path}] @
+ [#:effective-version "2.2"] @
[#:references-graphs #f] [#:allowed-references #f] @
[#:disallowed-references #f] @
[#:leaked-env-vars #f] @
@@ -5181,6 +5211,9 @@ make @var{modules} available in the evaluation context of @var{exp};
the load path during the execution of @var{exp}---e.g., @code{((guix
build utils) (guix build gnu-build-system))}.
+@var{effective-version} determines the string to use when adding extensions of
+@var{exp} (see @code{with-extensions}) to the search path---e.g., @code{"2.2"}.
+
@var{graft?} determines whether packages referred to by @var{exp} should be grafted when
applicable.
diff --git a/guix/gexp.scm b/guix/gexp.scm
index fdfd734245..338c339da9 100644
--- a/guix/gexp.scm
+++ b/guix/gexp.scm
@@ -33,6 +33,7 @@
#:export (gexp
gexp?
with-imported-modules
+ with-extensions
gexp-input
gexp-input?
@@ -118,10 +119,11 @@
;; "G expressions".
(define-record-type <gexp>
- (make-gexp references modules proc)
+ (make-gexp references modules extensions proc)
gexp?
(references gexp-references) ;list of <gexp-input>
(modules gexp-self-modules) ;list of module names
+ (extensions gexp-self-extensions) ;list of lowerable things
(proc gexp-proc)) ;procedure
(define (write-gexp gexp port)
@@ -492,19 +494,20 @@ whether this should be considered a \"native\" input or not."
(set-record-type-printer! <gexp-output> write-gexp-output)
-(define (gexp-modules gexp)
- "Return the list of Guile module names GEXP relies on. If (gexp? GEXP) is
-false, meaning that GEXP is a plain Scheme object, return the empty list."
+(define (gexp-attribute gexp self-attribute)
+ "Recurse on GEXP and the expressions it refers to, summing the items
+returned by SELF-ATTRIBUTE, a procedure that takes a gexp."
(if (gexp? gexp)
(delete-duplicates
- (append (gexp-self-modules gexp)
+ (append (self-attribute gexp)
(append-map (match-lambda
(($ <gexp-input> (? gexp? exp))
- (gexp-modules exp))
+ (gexp-attribute exp self-attribute))
(($ <gexp-input> (lst ...))
(append-map (lambda (item)
(if (gexp? item)
- (gexp-modules item)
+ (gexp-attribute item
+ self-attribute)
'()))
lst))
(_
@@ -512,6 +515,17 @@ false, meaning that GEXP is a plain Scheme object, return the empty list."
(gexp-references gexp))))
'())) ;plain Scheme data type
+(define (gexp-modules gexp)
+ "Return the list of Guile module names GEXP relies on. If (gexp? GEXP) is
+false, meaning that GEXP is a plain Scheme object, return the empty list."
+ (gexp-attribute gexp gexp-self-modules))
+
+(define (gexp-extensions gexp)
+ "Return the list of Guile extensions (packages) GEXP relies on. If (gexp?
+GEXP) is false, meaning that GEXP is a plain Scheme object, return the empty
+list."
+ (gexp-attribute gexp gexp-self-extensions))
+
(define* (lower-inputs inputs
#:key system target)
"Turn any package from INPUTS into a derivation for SYSTEM; return the
@@ -577,6 +591,7 @@ names and file names suitable for the #:allowed-references argument to
(modules '())
(module-path %load-path)
(guile-for-build (%guile-for-build))
+ (effective-version "2.2")
(graft? (%graft?))
references-graphs
allowed-references disallowed-references
@@ -595,6 +610,9 @@ names of Guile modules searched in MODULE-PATH to be copied in the store,
compiled, and made available in the load path during the execution of
EXP---e.g., '((guix build utils) (guix build gnu-build-system)).
+EFFECTIVE-VERSION determines the string to use when adding extensions of
+EXP (see 'with-extensions') to the search path---e.g., \"2.2\".
+
GRAFT? determines whether packages referred to by EXP should be grafted when
applicable.
@@ -630,7 +648,7 @@ The other arguments are as for 'derivation'."
(define (graphs-file-names graphs)
;; Return a list of (FILE-NAME . STORE-PATH) pairs made from GRAPHS.
(map (match-lambda
- ;; TODO: Remove 'derivation?' special cases.
+ ;; TODO: Remove 'derivation?' special cases.
((file-name (? derivation? drv))
(cons file-name (derivation->output-path drv)))
((file-name (? derivation? drv) sub-drv)
@@ -639,7 +657,13 @@ The other arguments are as for 'derivation'."
(cons file-name thing)))
graphs))
- (mlet* %store-monad (;; The following binding forces '%current-system' and
+ (define (extension-flags extension)
+ `("-L" ,(string-append (derivation->output-path extension)
+ "/share/guile/site/" effective-version)
+ "-C" ,(string-append (derivation->output-path extension)
+ "/lib/guile/" effective-version "/site-ccache")))
+
+ (mlet* %store-monad ( ;; The following binding forces '%current-system' and
;; '%current-target-system' to be looked up at >>=
;; time.
(graft? (set-grafting graft?))
@@ -660,6 +684,11 @@ The other arguments are as for 'derivation'."
#:target target))
(builder (text-file script-name
(object->string sexp)))
+ (extensions -> (gexp-extensions exp))
+ (exts (mapm %store-monad
+ (lambda (obj)
+ (lower-object obj system))
+ extensions))
(modules (if (pair? %modules)
(imported-modules %modules
#:system system
@@ -672,6 +701,7 @@ The other arguments are as for 'derivation'."
(compiled-modules %modules
#:system system
#:module-path module-path
+ #:extensions extensions
#:guile guile-for-build
#:deprecation-warnings
deprecation-warnings)
@@ -704,6 +734,7 @@ The other arguments are as for 'derivation'."
`("-L" ,(derivation->output-path modules)
"-C" ,(derivation->output-path compiled))
'())
+ ,@(append-map extension-flags exts)
,builder)
#:outputs outputs
#:env-vars env-vars
@@ -713,6 +744,7 @@ The other arguments are as for 'derivation'."
,@(if modules
`((,modules) (,compiled) ,@inputs)
inputs)
+ ,@(map list exts)
,@(match graphs
(((_ . inputs) ...) inputs)
(_ '())))
@@ -861,6 +893,17 @@ environment."
(identifier-syntax modules)))
body ...))
+(define-syntax-parameter current-imported-extensions
+ ;; Current list of extensions.
+ (identifier-syntax '()))
+
+(define-syntax-rule (with-extensions extensions body ...)
+ "Mark the gexps defined in BODY... as requiring EXTENSIONS in their
+execution environment."
+ (syntax-parameterize ((current-imported-extensions
+ (identifier-syntax extensions)))
+ body ...))
+
(define-syntax gexp
(lambda (s)
(define (collect-escapes exp)
@@ -957,6 +1000,7 @@ environment."
(refs (map escape->ref escapes)))
#`(make-gexp (list #,@refs)
current-imported-modules
+ current-imported-extensions
(lambda #,formals
#,sexp)))))))
@@ -1071,6 +1115,7 @@ last one is created from the given <scheme-file> object."
(system (%current-system))
(guile (%guile-for-build))
(module-path %load-path)
+ (extensions '())
(deprecation-warnings #f))
"Return a derivation that builds a tree containing the `.go' files
corresponding to MODULES. All the MODULES are built in a context where
@@ -1129,6 +1174,26 @@ they can refer to each other."
(@ (guix build utils) mkdir-p))))
'()))
+ ;; Add EXTENSIONS to the search path.
+ ;; TODO: Remove the outer 'ungexp-splicing' on the next rebuild cycle.
+ (ungexp-splicing
+ (if (null? extensions)
+ '()
+ (gexp ((set! %load-path
+ (append (map (lambda (extension)
+ (string-append extension
+ "/share/guile/site/"
+ (effective-version)))
+ '((ungexp-native-splicing extensions)))
+ %load-path))
+ (set! %load-compiled-path
+ (append (map (lambda (extension)
+ (string-append extension "/lib/guile/"
+ (effective-version)
+ "/site-ccache"))
+ '((ungexp-native-splicing extensions)))
+ %load-compiled-path))))))
+
(set! %load-path (cons (ungexp modules) %load-path))
(ungexp-splicing
@@ -1174,20 +1239,34 @@ they can refer to each other."
(module-ref (resolve-interface '(gnu packages guile))
'guile-2.2))
-(define* (load-path-expression modules #:optional (path %load-path))
+(define* (load-path-expression modules #:optional (path %load-path)
+ #:key (extensions '()))
"Return as a monadic value a gexp that sets '%load-path' and
'%load-compiled-path' to point to MODULES, a list of module names. MODULES
are searched for in PATH."
(mlet %store-monad ((modules (imported-modules modules
#:module-path path))
(compiled (compiled-modules modules
+ #:extensions extensions
#:module-path path)))
(return (gexp (eval-when (expand load eval)
(set! %load-path
- (cons (ungexp modules) %load-path))
+ (cons (ungexp modules)
+ (append (map (lambda (extension)
+ (string-append extension
+ "/share/guile/site/"
+ (effective-version)))
+ '((ungexp-native-splicing extensions)))
+ %load-path)))
(set! %load-compiled-path
(cons (ungexp compiled)
- %load-compiled-path)))))))
+ (append (map (lambda (extension)
+ (string-append extension
+ "/lib/guile/"
+ (effective-version)
+ "/site-ccache"))
+ '((ungexp-native-splicing extensions)))
+ %load-compiled-path))))))))
(define* (gexp->script name exp
#:key (guile (default-guile))
@@ -1196,7 +1275,9 @@ are searched for in PATH."
imported modules in its search path. Look up EXP's modules in MODULE-PATH."
(mlet %store-monad ((set-load-path
(load-path-expression (gexp-modules exp)
- module-path)))
+ module-path
+ #:extensions
+ (gexp-extensions exp))))
(gexp->derivation name
(gexp
(call-with-output-file (ungexp output)
@@ -1225,35 +1306,38 @@ the resulting file.
When SET-LOAD-PATH? is true, emit code in the resulting file to set
'%load-path' and '%load-compiled-path' to honor EXP's imported modules.
Lookup EXP's modules in MODULE-PATH."
- (match (if set-load-path? (gexp-modules exp) '())
- (() ;zero modules
- (gexp->derivation name
- (gexp
- (call-with-output-file (ungexp output)
- (lambda (port)
- (for-each (lambda (exp)
- (write exp port))
- '(ungexp (if splice?
- exp
- (gexp ((ungexp exp)))))))))
- #:local-build? #t
- #:substitutable? #f))
- ((modules ...)
- (mlet %store-monad ((set-load-path (load-path-expression modules
- module-path)))
- (gexp->derivation name
- (gexp
- (call-with-output-file (ungexp output)
- (lambda (port)
- (write '(ungexp set-load-path) port)
- (for-each (lambda (exp)
- (write exp port))
- '(ungexp (if splice?
- exp
- (gexp ((ungexp exp)))))))))
- #:module-path module-path
- #:local-build? #t
- #:substitutable? #f)))))
+ (define modules (gexp-modules exp))
+ (define extensions (gexp-extensions exp))
+
+ (if (or (not set-load-path?)
+ (and (null? modules) (null? extensions)))
+ (gexp->derivation name
+ (gexp
+ (call-with-output-file (ungexp output)
+ (lambda (port)
+ (for-each (lambda (exp)
+ (write exp port))
+ '(ungexp (if splice?
+ exp
+ (gexp ((ungexp exp)))))))))
+ #:local-build? #t
+ #:substitutable? #f)
+ (mlet %store-monad ((set-load-path
+ (load-path-expression modules module-path
+ #:extensions extensions)))
+ (gexp->derivation name
+ (gexp
+ (call-with-output-file (ungexp output)
+ (lambda (port)
+ (write '(ungexp set-load-path) port)
+ (for-each (lambda (exp)
+ (write exp port))
+ '(ungexp (if splice?
+ exp
+ (gexp ((ungexp exp)))))))))
+ #:module-path module-path
+ #:local-build? #t
+ #:substitutable? #f))))
(define* (text-file* name #:rest text)
"Return as a monadic value a derivation that builds a text file containing
diff --git a/tests/gexp.scm b/tests/gexp.scm
index 3c8b4624da..a560adfc5c 100644
--- a/tests/gexp.scm
+++ b/tests/gexp.scm
@@ -23,6 +23,7 @@
#:use-module (guix grafts)
#:use-module (guix derivations)
#:use-module (guix packages)
+ #:use-module (guix build-system trivial)
#:use-module (guix tests)
#:use-module ((guix build utils) #:select (with-directory-excursion))
#:use-module ((guix utils) #:select (call-with-temporary-directory))
@@ -66,6 +67,27 @@
(run-with-store %store exp
#:guile-for-build (%guile-for-build))))
+(define %extension-package
+ ;; Example of a package to use when testing 'with-extensions'.
+ (dummy-package "extension"
+ (build-system trivial-build-system)
+ (arguments
+ `(#:guile ,%bootstrap-guile
+ #:modules ((guix build utils))
+ #:builder
+ (begin
+ (use-modules (guix build utils))
+ (let* ((out (string-append (assoc-ref %outputs "out")
+ "/share/guile/site/"
+ (effective-version))))
+ (mkdir-p out)
+ (call-with-output-file (string-append out "/hg2g.scm")
+ (lambda (port)
+ (write '(define-module (hg2g)
+ #:export (the-answer))
+ port)
+ (write '(define the-answer 42) port)))))))))
+
(test-begin "gexp")
@@ -739,6 +761,54 @@
(built-derivations (list drv))
(return (= 42 (call-with-input-file out read))))))
+(test-equal "gexp-extensions & ungexp"
+ (list sed grep)
+ ((@@ (guix gexp) gexp-extensions)
+ #~(foo #$(with-extensions (list grep) #~+)
+ #+(with-extensions (list sed) #~-))))
+
+(test-equal "gexp-extensions & ungexp-splicing"
+ (list grep sed)
+ ((@@ (guix gexp) gexp-extensions)
+ #~(foo #$@(list (with-extensions (list grep) #~+)
+ (with-imported-modules '((foo))
+ (with-extensions (list sed) #~-))))))
+
+(test-equal "gexp-extensions and literal Scheme object"
+ '()
+ ((@@ (guix gexp) gexp-extensions) #t))
+
+(test-assertm "gexp->derivation & with-extensions"
+ ;; Create a fake Guile extension and make sure it is accessible both to the
+ ;; imported modules and to the derivation build script.
+ (mlet* %store-monad
+ ((extension -> %extension-package)
+ (module -> (scheme-file "x" #~( ;; splice!
+ (define-module (foo)
+ #:use-module (hg2g)
+ #:export (multiply))
+
+ (define (multiply x)
+ (* the-answer x)))
+ #:splice? #t))
+ (build -> (with-extensions (list extension)
+ (with-imported-modules `((guix build utils)
+ ((foo) => ,module))
+ #~(begin
+ (use-modules (guix build utils)
+ (hg2g) (foo))
+ (call-with-output-file #$output
+ (lambda (port)
+ (write (list the-answer (multiply 2))
+ port)))))))
+ (drv (gexp->derivation "thingie" build
+ ;; %BOOTSTRAP-GUILE is 2.0.
+ #:effective-version "2.0"))
+ (out -> (derivation->output-path drv)))
+ (mbegin %store-monad
+ (built-derivations (list drv))
+ (return (equal? '(42 84) (call-with-input-file out read))))))
+
(test-assertm "gexp->derivation #:references-graphs"
(mlet* %store-monad
((one (text-file "one" (random-text)))
@@ -948,6 +1018,22 @@
(return (and (zero? (close-pipe pipe))
(string=? text str))))))))))
+(test-assertm "program-file & with-extensions"
+ (let* ((exp (with-extensions (list %extension-package)
+ (gexp (begin
+ (use-modules (hg2g))
+ (display the-answer)))))
+ (file (program-file "program" exp
+ #:guile %bootstrap-guile)))
+ (mlet* %store-monad ((drv (lower-object file))
+ (out -> (derivation->output-path drv)))
+ (mbegin %store-monad
+ (built-derivations (list drv))
+ (let* ((pipe (open-input-pipe out))
+ (str (get-string-all pipe)))
+ (return (and (zero? (close-pipe pipe))
+ (= 42 (string->number str)))))))))
+
(test-assertm "scheme-file"
(let* ((text (plain-file "foo" "Hello, world!"))
(scheme (scheme-file "bar" #~(list "foo" #$text))))
--
cgit 1.4.1
From 7f9d184d9b688d13ce76eefabaddcfa76bdde2b5 Mon Sep 17 00:00:00 2001
From: Caleb Ristvedt <caleb.ristvedt@cune.org>
Date: Sun, 27 May 2018 19:19:30 +0200
Subject: Add (gnu store database).
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
* guix/config.scm.in (%store-database-directory): New variable.
* guix/store/database.scm: New file.
* tests/store-database.scm: New file.
* Makefile.am (STORE_MODULES): New variable.
(MODULES, MODULES_NOT_COMPILED): Adjust accordingly.
(SCM_TESTS) [HAVE_GUILE_SQLITE3]: Add tests/store-database.scm.
Co-authored-by: Ludovic Courtès <ludo@gnu.org>
---
.dir-locals.el | 2 +
Makefile.am | 17 ++++
guix/config.scm.in | 6 ++
guix/store/database.scm | 234 +++++++++++++++++++++++++++++++++++++++++++++++
tests/store-database.scm | 54 +++++++++++
5 files changed, 313 insertions(+)
create mode 100644 guix/store/database.scm
create mode 100644 tests/store-database.scm
(limited to 'tests')
diff --git a/.dir-locals.el b/.dir-locals.el
index 2db751ca22..eb99a5bcc1 100644
--- a/.dir-locals.el
+++ b/.dir-locals.el
@@ -75,6 +75,8 @@
(eval . (put 'with-imported-modules 'scheme-indent-function 1))
(eval . (put 'with-extensions 'scheme-indent-function 1))
+ (eval . (put 'with-database 'scheme-indent-function 2))
+
(eval . (put 'call-with-container 'scheme-indent-function 1))
(eval . (put 'container-excursion 'scheme-indent-function 1))
(eval . (put 'eventually 'scheme-indent-function 1))
diff --git a/Makefile.am b/Makefile.am
index 2a0a858429..d81fce5585 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -257,6 +257,16 @@ MODULES += \
endif BUILD_DAEMON_OFFLOAD
+# Scheme implementation of the build daemon and related functionality.
+STORE_MODULES = \
+ guix/store/database.scm
+
+if HAVE_GUILE_SQLITE3
+MODULES += $(STORE_MODULES)
+else
+MODULES_NOT_COMPILED += $(STORE_MODULES)
+endif !HAVE_GUILE_SQLITE3
+
# Internal modules with test suite support.
dist_noinst_DATA = guix/tests.scm guix/tests/http.scm
@@ -379,6 +389,13 @@ SCM_TESTS += \
endif
+if HAVE_GUILE_SQLITE3
+
+SCM_TESTS += \
+ tests/store-database.scm
+
+endif
+
SH_TESTS = \
tests/guix-build.sh \
tests/guix-download.sh \
diff --git a/guix/config.scm.in b/guix/config.scm.in
index 8f2c4abd8e..dfe5fe0dbf 100644
--- a/guix/config.scm.in
+++ b/guix/config.scm.in
@@ -1,5 +1,6 @@
;;; GNU Guix --- Functional package management for GNU
;;; Copyright © 2012, 2013, 2014, 2015, 2016 Ludovic Courtès <ludo@gnu.org>
+;;; Copyright © 2017 Caleb Ristvedt <caleb.ristvedt@cune.org>
;;;
;;; This file is part of GNU Guix.
;;;
@@ -29,6 +30,7 @@
%store-directory
%state-directory
+ %store-database-directory
%config-directory
%guix-register-program
@@ -80,6 +82,10 @@
(or (getenv "NIX_STATE_DIR")
(string-append %localstatedir "/guix")))
+(define %store-database-directory
+ (or (and=> (getenv "NIX_DB_DIR") canonicalize-path)
+ (string-append %state-directory "/db")))
+
(define %config-directory
;; This must match `GUIX_CONFIGURATION_DIRECTORY' as defined in `nix/local.mk'.
(or (getenv "GUIX_CONFIGURATION_DIRECTORY")
diff --git a/guix/store/database.scm b/guix/store/database.scm
new file mode 100644
index 0000000000..4233219ba0
--- /dev/null
+++ b/guix/store/database.scm
@@ -0,0 +1,234 @@
+;;; GNU Guix --- Functional package management for GNU
+;;; Copyright © 2017 Caleb Ristvedt <caleb.ristvedt@cune.org>
+;;; Copyright © 2018 Ludovic Courtès <ludo@gnu.org>
+;;;
+;;; This file is part of GNU Guix.
+;;;
+;;; GNU Guix is free software; you can redistribute it and/or modify it
+;;; under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3 of the License, or (at
+;;; your option) any later version.
+;;;
+;;; GNU Guix is distributed in the hope that it will be useful, but
+;;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
+
+(define-module (guix store database)
+ #:use-module (sqlite3)
+ #:use-module (guix config)
+ #:use-module (guix serialization)
+ #:use-module (guix base16)
+ #:use-module (guix hash)
+ #:use-module (rnrs io ports)
+ #:use-module (srfi srfi-11)
+ #:use-module (srfi srfi-19)
+ #:use-module (ice-9 match)
+ #:export (sqlite-register
+ register-path))
+
+;;; Code for working with the store database directly.
+
+
+(define-syntax-rule (with-database file db exp ...)
+ "Open DB from FILE and close it when the dynamic extent of EXP... is left."
+ (let ((db (sqlite-open file)))
+ (dynamic-wind noop
+ (lambda ()
+ exp ...)
+ (lambda ()
+ (sqlite-close db)))))
+
+(define (last-insert-row-id db)
+ ;; XXX: (sqlite3) currently lacks bindings for 'sqlite3_last_insert_rowid'.
+ ;; Work around that.
+ (let* ((stmt (sqlite-prepare db "SELECT last_insert_rowid();"
+ #:cache? #t))
+ (result (sqlite-fold cons '() stmt)))
+ (sqlite-finalize stmt)
+ (match result
+ ((#(id)) id)
+ (_ #f))))
+
+(define path-id-sql
+ "SELECT id FROM ValidPaths WHERE path = :path")
+
+(define* (path-id db path)
+ "If PATH exists in the 'ValidPaths' table, return its numerical
+identifier. Otherwise, return #f."
+ (let ((stmt (sqlite-prepare db path-id-sql #:cache? #t)))
+ (sqlite-bind-arguments stmt #:path path)
+ (let ((result (sqlite-fold cons '() stmt)))
+ (sqlite-finalize stmt)
+ (match result
+ ((#(id) . _) id)
+ (_ #f)))))
+
+(define update-sql
+ "UPDATE ValidPaths SET hash = :hash, registrationTime = :time, deriver =
+:deriver, narSize = :size WHERE id = :id")
+
+(define insert-sql
+ "INSERT INTO ValidPaths (path, hash, registrationTime, deriver, narSize)
+VALUES (:path, :hash, :time, :deriver, :size)")
+
+(define* (update-or-insert db #:key path deriver hash nar-size time)
+ "The classic update-if-exists and insert-if-doesn't feature that sqlite
+doesn't exactly have... they've got something close, but it involves deleting
+and re-inserting instead of updating, which causes problems with foreign keys,
+of course. Returns the row id of the row that was modified or inserted."
+ (let ((id (path-id db path)))
+ (if id
+ (let ((stmt (sqlite-prepare db update-sql #:cache? #t)))
+ (sqlite-bind-arguments stmt #:id id
+ #:path path #:deriver deriver
+ #:hash hash #:size nar-size #:time time)
+ (sqlite-fold cons '() stmt)
+ (sqlite-finalize stmt)
+ (last-insert-row-id db))
+ (let ((stmt (sqlite-prepare db insert-sql #:cache? #t)))
+ (sqlite-bind-arguments stmt
+ #:path path #:deriver deriver
+ #:hash hash #:size nar-size #:time time)
+ (sqlite-fold cons '() stmt) ;execute it
+ (sqlite-finalize stmt)
+ (last-insert-row-id db)))))
+
+(define add-reference-sql
+ "INSERT OR IGNORE INTO Refs (referrer, reference) SELECT :referrer, id
+FROM ValidPaths WHERE path = :reference")
+
+(define (add-references db referrer references)
+ "REFERRER is the id of the referring store item, REFERENCES is a list
+containing store items being referred to. Note that all of the store items in
+REFERENCES must already be registered."
+ (let ((stmt (sqlite-prepare db add-reference-sql #:cache? #t)))
+ (for-each (lambda (reference)
+ (sqlite-reset stmt)
+ (sqlite-bind-arguments stmt #:referrer referrer
+ #:reference reference)
+ (sqlite-fold cons '() stmt) ;execute it
+ (sqlite-finalize stmt)
+ (last-insert-row-id db))
+ references)))
+
+;; XXX figure out caching of statement and database objects... later
+(define* (sqlite-register #:key db-file path (references '())
+ deriver hash nar-size)
+ "Registers this stuff in a database specified by DB-FILE. PATH is the string
+path of some store item, REFERENCES is a list of string paths which the store
+item PATH refers to (they need to be already registered!), DERIVER is a string
+path of the derivation that created the store item PATH, HASH is the
+base16-encoded sha256 hash of the store item denoted by PATH (prefixed with
+\"sha256:\") after being converted to nar form, and nar-size is the size in
+bytes of the store item denoted by PATH after being converted to nar form."
+ (with-database db-file db
+ (let ((id (update-or-insert db #:path path
+ #:deriver deriver
+ #:hash hash
+ #:nar-size nar-size
+ #:time (time-second (current-time time-utc)))))
+ (add-references db id references))))
+
+
+;;;
+;;; High-level interface.
+;;;
+
+;; XXX: Would it be better to just make WRITE-FILE give size as well? I question
+;; the general utility of this approach.
+(define (counting-wrapper-port output-port)
+ "Some custom ports don't implement GET-POSITION at all. But if we want to
+figure out how many bytes are being written, we will want to use that. So this
+makes a wrapper around a port which implements GET-POSITION."
+ (let ((byte-count 0))
+ (make-custom-binary-output-port "counting-wrapper"
+ (lambda (bytes offset count)
+ (set! byte-count
+ (+ byte-count count))
+ (put-bytevector output-port bytes
+ offset count)
+ count)
+ (lambda ()
+ byte-count)
+ #f
+ (lambda ()
+ (close-port output-port)))))
+
+
+(define (nar-sha256 file)
+ "Gives the sha256 hash of a file and the size of the file in nar form."
+ (let-values (((port get-hash) (open-sha256-port)))
+ (let ((wrapper (counting-wrapper-port port)))
+ (write-file file wrapper)
+ (force-output wrapper)
+ (force-output port)
+ (let ((hash (get-hash))
+ (size (port-position wrapper)))
+ (close-port wrapper)
+ (values hash size)))))
+
+;; TODO: make this canonicalize store items that are registered. This involves
+;; setting permissions and timestamps, I think. Also, run a "deduplication
+;; pass", whatever that involves. Also, handle databases not existing yet
+;; (what should the default behavior be? Figuring out how the C++ stuff
+;; currently does it sounds like a lot of grepping for global
+;; variables...). Also, return #t on success like the documentation says we
+;; should.
+
+(define* (register-path path
+ #:key (references '()) deriver prefix
+ state-directory)
+ ;; Priority for options: first what is given, then environment variables,
+ ;; then defaults. %state-directory, %store-directory, and
+ ;; %store-database-directory already handle the "environment variables /
+ ;; defaults" question, so we only need to choose between what is given and
+ ;; those.
+ "Register PATH as a valid store file, with REFERENCES as its list of
+references, and DERIVER as its deriver (.drv that led to it.) If PREFIX is
+given, it must be the name of the directory containing the new store to
+initialize; if STATE-DIRECTORY is given, it must be a string containing the
+absolute file name to the state directory of the store being initialized.
+Return #t on success.
+
+Use with care as it directly modifies the store! This is primarily meant to
+be used internally by the daemon's build hook."
+ (let* ((db-dir (cond
+ (state-directory
+ (string-append state-directory "/db"))
+ (prefix
+ ;; If prefix is specified, the value of NIX_STATE_DIR
+ ;; (which affects %state-directory) isn't supposed to
+ ;; affect db-dir, only the compile-time-customized
+ ;; default should.
+ (string-append prefix %localstatedir "/guix/db"))
+ (else
+ %store-database-directory)))
+ (store-dir (if prefix
+ ;; same situation as above
+ (string-append prefix %storedir)
+ %store-directory))
+ (to-register (if prefix
+ (string-append %storedir "/" (basename path))
+ ;; note: we assume here that if path is, for
+ ;; example, /foo/bar/gnu/store/thing.txt and prefix
+ ;; isn't given, then an environment variable has
+ ;; been used to change the store directory to
+ ;; /foo/bar/gnu/store, since otherwise real-path
+ ;; would end up being /gnu/store/thing.txt, which is
+ ;; probably not the right file in this case.
+ path))
+ (real-path (string-append store-dir "/" (basename path))))
+ (let-values (((hash nar-size)
+ (nar-sha256 real-path)))
+ (sqlite-register
+ #:db-file (string-append db-dir "/db.sqlite")
+ #:path to-register
+ #:references references
+ #:deriver deriver
+ #:hash (string-append "sha256:"
+ (bytevector->base16-string hash))
+ #:nar-size nar-size))))
diff --git a/tests/store-database.scm b/tests/store-database.scm
new file mode 100644
index 0000000000..1348a75c26
--- /dev/null
+++ b/tests/store-database.scm
@@ -0,0 +1,54 @@
+;;; GNU Guix --- Functional package management for GNU
+;;; Copyright © 2017, 2018 Ludovic Courtès <ludo@gnu.org>
+;;;
+;;; This file is part of GNU Guix.
+;;;
+;;; GNU Guix is free software; you can redistribute it and/or modify it
+;;; under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3 of the License, or (at
+;;; your option) any later version.
+;;;
+;;; GNU Guix is distributed in the hope that it will be useful, but
+;;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
+
+(define-module (test-store-database)
+ #:use-module (guix tests)
+ #:use-module ((guix store) #:hide (register-path))
+ #:use-module (guix store database)
+ #:use-module (srfi srfi-26)
+ #:use-module (srfi srfi-64))
+
+;; Test the (guix store database) module.
+
+(define %store
+ (open-connection-for-tests))
+
+
+(test-begin "store-database")
+
+(test-assert "register-path"
+ (let ((file (string-append (%store-prefix) "/" (make-string 32 #\f)
+ "-fake")))
+ (when (valid-path? %store file)
+ (delete-paths %store (list file)))
+ (false-if-exception (delete-file file))
+
+ (let ((ref (add-text-to-store %store "ref-of-fake" (random-text)))
+ (drv (string-append file ".drv")))
+ (call-with-output-file file
+ (cut display "This is a fake store item.\n" <>))
+ (register-path file
+ #:references (list ref)
+ #:deriver drv)
+
+ (and (valid-path? %store file)
+ (equal? (references %store file) (list ref))
+ (null? (valid-derivers %store file))
+ (null? (referrers %store file))))))
+
+(test-end "store-database")
--
cgit 1.4.1
From bf5bf5778cb7c3a2475c6acd707abc925b1819aa Mon Sep 17 00:00:00 2001
From: Caleb Ristvedt <caleb.ristvedt@cune.org>
Date: Sun, 27 May 2018 23:20:54 +0200
Subject: Add (guix store deduplication).
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
* guix/store/database.scm (register-path): Add #:deduplicate? and call
'deduplicate' when it's true.
(counting-wrapper-port, nar-sha256): Move to...
* guix/store/deduplication.scm: ... here. New file.
* tests/store-deduplication.scm: New file.
* Makefile.am (STORE_MODULES): Add deduplication.scm.
(SCM_TESTS) [HAVE_GUILE_SQLITE3]: Add store-deduplication.scm.
Co-authored-by: Ludovic Courtès <ludo@gnu.org>
---
Makefile.am | 6 +-
guix/store/database.scm | 43 ++----------
guix/store/deduplication.scm | 148 ++++++++++++++++++++++++++++++++++++++++++
tests/store-deduplication.scm | 64 ++++++++++++++++++
4 files changed, 222 insertions(+), 39 deletions(-)
create mode 100644 guix/store/deduplication.scm
create mode 100644 tests/store-deduplication.scm
(limited to 'tests')
diff --git a/Makefile.am b/Makefile.am
index d81fce5585..474575c9f2 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -259,7 +259,8 @@ endif BUILD_DAEMON_OFFLOAD
# Scheme implementation of the build daemon and related functionality.
STORE_MODULES = \
- guix/store/database.scm
+ guix/store/database.scm \
+ guix/store/deduplication.scm
if HAVE_GUILE_SQLITE3
MODULES += $(STORE_MODULES)
@@ -392,7 +393,8 @@ endif
if HAVE_GUILE_SQLITE3
SCM_TESTS += \
- tests/store-database.scm
+ tests/store-database.scm \
+ tests/store-deduplication.scm
endif
diff --git a/guix/store/database.scm b/guix/store/database.scm
index b9745dbe14..3623c0e7a0 100644
--- a/guix/store/database.scm
+++ b/guix/store/database.scm
@@ -21,10 +21,9 @@
#:use-module (sqlite3)
#:use-module (guix config)
#:use-module (guix serialization)
+ #:use-module (guix store deduplication)
#:use-module (guix base16)
- #:use-module (guix hash)
#:use-module (guix build syscalls)
- #:use-module (rnrs io ports)
#:use-module (srfi srfi-11)
#:use-module (srfi srfi-19)
#:use-module (ice-9 match)
@@ -140,39 +139,6 @@ bytes of the store item denoted by PATH after being converted to nar form."
;;; High-level interface.
;;;
-;; XXX: Would it be better to just make WRITE-FILE give size as well? I question
-;; the general utility of this approach.
-(define (counting-wrapper-port output-port)
- "Some custom ports don't implement GET-POSITION at all. But if we want to
-figure out how many bytes are being written, we will want to use that. So this
-makes a wrapper around a port which implements GET-POSITION."
- (let ((byte-count 0))
- (make-custom-binary-output-port "counting-wrapper"
- (lambda (bytes offset count)
- (set! byte-count
- (+ byte-count count))
- (put-bytevector output-port bytes
- offset count)
- count)
- (lambda ()
- byte-count)
- #f
- (lambda ()
- (close-port output-port)))))
-
-
-(define (nar-sha256 file)
- "Gives the sha256 hash of a file and the size of the file in nar form."
- (let-values (((port get-hash) (open-sha256-port)))
- (let ((wrapper (counting-wrapper-port port)))
- (write-file file wrapper)
- (force-output wrapper)
- (force-output port)
- (let ((hash (get-hash))
- (size (port-position wrapper)))
- (close-port wrapper)
- (values hash size)))))
-
;; TODO: Factorize with that in (gnu build install).
(define (reset-timestamps file)
"Reset the modification time on FILE and on all the files it contains, if
@@ -211,7 +177,7 @@ it's a directory."
(define* (register-path path
#:key (references '()) deriver prefix
- state-directory)
+ state-directory (deduplicate? #t))
;; Priority for options: first what is given, then environment variables,
;; then defaults. %state-directory, %store-directory, and
;; %store-database-directory already handle the "environment variables /
@@ -262,4 +228,7 @@ be used internally by the daemon's build hook."
#:deriver deriver
#:hash (string-append "sha256:"
(bytevector->base16-string hash))
- #:nar-size nar-size))))
+ #:nar-size nar-size)
+
+ (when deduplicate?
+ (deduplicate real-path hash #:store store-dir)))))
diff --git a/guix/store/deduplication.scm b/guix/store/deduplication.scm
new file mode 100644
index 0000000000..4b4ac01f64
--- /dev/null
+++ b/guix/store/deduplication.scm
@@ -0,0 +1,148 @@
+;;; GNU Guix --- Functional package management for GNU
+;;; Copyright © 2017 Caleb Ristvedt <caleb.ristvedt@cune.org>
+;;; Copyright © 2018 Ludovic Courtès <ludo@gnu.org>
+;;;
+;;; This file is part of GNU Guix.
+;;;
+;;; GNU Guix is free software; you can redistribute it and/or modify it
+;;; under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3 of the License, or (at
+;;; your option) any later version.
+;;;
+;;; GNU Guix is distributed in the hope that it will be useful, but
+;;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
+
+;;; This houses stuff we do to files when they arrive at the store - resetting
+;;; timestamps, deduplicating, etc.
+
+(define-module (guix store deduplication)
+ #:use-module (guix hash)
+ #:use-module (guix build utils)
+ #:use-module (guix base16)
+ #:use-module (srfi srfi-11)
+ #:use-module (rnrs io ports)
+ #:use-module (ice-9 ftw)
+ #:use-module (guix serialization)
+ #:export (nar-sha256
+ deduplicate))
+
+;; Would it be better to just make WRITE-FILE give size as well? I question
+;; the general utility of this approach.
+(define (counting-wrapper-port output-port)
+ "Some custom ports don't implement GET-POSITION at all. But if we want to
+figure out how many bytes are being written, we will want to use that. So this
+makes a wrapper around a port which implements GET-POSITION."
+ (let ((byte-count 0))
+ (make-custom-binary-output-port "counting-wrapper"
+ (lambda (bytes offset count)
+ (set! byte-count
+ (+ byte-count count))
+ (put-bytevector output-port bytes
+ offset count)
+ count)
+ (lambda ()
+ byte-count)
+ #f
+ (lambda ()
+ (close-port output-port)))))
+
+(define (nar-sha256 file)
+ "Gives the sha256 hash of a file and the size of the file in nar form."
+ (let-values (((port get-hash) (open-sha256-port)))
+ (let ((wrapper (counting-wrapper-port port)))
+ (write-file file wrapper)
+ (force-output wrapper)
+ (force-output port)
+ (let ((hash (get-hash))
+ (size (port-position wrapper)))
+ (close-port wrapper)
+ (values hash size)))))
+
+(define (tempname-in directory)
+ "Gives an unused temporary name under DIRECTORY. Not guaranteed to still be
+unused by the time you create anything with that name, but a good shot."
+ (let ((const-part (string-append directory "/.tmp-link-"
+ (number->string (getpid)))))
+ (let try ((guess-part
+ (number->string (random most-positive-fixnum) 16)))
+ (if (file-exists? (string-append const-part "-" guess-part))
+ (try (number->string (random most-positive-fixnum) 16))
+ (string-append const-part "-" guess-part)))))
+
+(define* (get-temp-link target #:optional (link-prefix (dirname target)))
+ "Like mkstemp!, but instead of creating a new file and giving you the name,
+it creates a new hardlink to TARGET and gives you the name. Since
+cross-filesystem hardlinks don't work, the temp link must be created on the
+same filesystem - where in that filesystem it is can be controlled by
+LINK-PREFIX."
+ (let try ((tempname (tempname-in link-prefix)))
+ (catch 'system-error
+ (lambda ()
+ (link target tempname)
+ tempname)
+ (lambda (args)
+ (if (= (system-error-errno args) EEXIST)
+ (try (tempname-in link-prefix))
+ (throw 'system-error args))))))
+
+;; There are 3 main kinds of errors we can get from hardlinking: "Too many
+;; things link to this" (EMLINK), "this link already exists" (EEXIST), and
+;; "can't fit more stuff in this directory" (ENOSPC).
+
+(define (replace-with-link target to-replace)
+ "Atomically replace the file TO-REPLACE with a link to TARGET. Note: TARGET
+and TO-REPLACE must be on the same file system."
+ (let ((temp-link (get-temp-link target (dirname to-replace))))
+ (rename-file temp-link to-replace)))
+
+(define-syntax-rule (false-if-system-error (errors ...) exp ...)
+ "Given ERRORS, a list of system error codes to ignore, evaluates EXP... and
+return #f if any of the system error codes in the given list are thrown."
+ (catch 'system-error
+ (lambda ()
+ exp ...)
+ (lambda args
+ (if (member (system-error-errno args) (list errors ...))
+ #f
+ (apply throw args)))))
+
+(define* (deduplicate path hash #:key (store %store-directory))
+ "Check if a store item with sha256 hash HASH already exists. If so,
+replace PATH with a hardlink to the already-existing one. If not, register
+PATH so that future duplicates can hardlink to it. PATH is assumed to be
+under STORE."
+ (let* ((links-directory (string-append store "/.links"))
+ (link-file (string-append links-directory "/"
+ (bytevector->base16-string hash))))
+ (mkdir-p links-directory)
+ (if (file-is-directory? path)
+ ;; Can't hardlink directories, so hardlink their atoms.
+ (for-each (lambda (file)
+ (unless (member file '("." ".."))
+ (deduplicate file (nar-sha256 file)
+ #:store store)))
+ (scandir path))
+ (if (file-exists? link-file)
+ (false-if-system-error (EMLINK)
+ (replace-with-link link-file path))
+ (catch 'system-error
+ (lambda ()
+ (link path link-file))
+ (lambda args
+ (let ((errno (system-error-errno args)))
+ (cond ((= errno EEXIST)
+ ;; Someone else put an entry for PATH in
+ ;; LINKS-DIRECTORY before we could. Let's use it.
+ (false-if-system-error (EMLINK)
+ (replace-with-link path link-file)))
+ ((= errno ENOSPC)
+ ;; There's not enough room in the directory index for
+ ;; more entries in .links, but that's fine: we can
+ ;; just stop.
+ #f)
+ (else (apply throw args))))))))))
diff --git a/tests/store-deduplication.scm b/tests/store-deduplication.scm
new file mode 100644
index 0000000000..04817a193a
--- /dev/null
+++ b/tests/store-deduplication.scm
@@ -0,0 +1,64 @@
+;;; GNU Guix --- Functional package management for GNU
+;;; Copyright © 2018 Ludovic Courtès <ludo@gnu.org>
+;;;
+;;; This file is part of GNU Guix.
+;;;
+;;; GNU Guix is free software; you can redistribute it and/or modify it
+;;; under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3 of the License, or (at
+;;; your option) any later version.
+;;;
+;;; GNU Guix is distributed in the hope that it will be useful, but
+;;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
+
+(define-module (test-store-deduplication)
+ #:use-module (guix tests)
+ #:use-module (guix store deduplication)
+ #:use-module (guix hash)
+ #:use-module ((guix utils) #:select (call-with-temporary-directory))
+ #:use-module (guix build utils)
+ #:use-module (rnrs bytevectors)
+ #:use-module (ice-9 binary-ports)
+ #:use-module (srfi srfi-1)
+ #:use-module (srfi srfi-64))
+
+(test-begin "store-deduplication")
+
+(test-equal "deduplicate"
+ (cons* #t #f ;inode comparisons
+ 2 (make-list 5 6)) ;'nlink' values
+
+ (call-with-temporary-directory
+ (lambda (store)
+ (let ((data (string->utf8 "Hello, world!"))
+ (identical (map (lambda (n)
+ (string-append store "/" (number->string n)))
+ (iota 5)))
+ (unique (string-append store "/unique")))
+ (for-each (lambda (file)
+ (call-with-output-file file
+ (lambda (port)
+ (put-bytevector port data))))
+ identical)
+ (call-with-output-file unique
+ (lambda (port)
+ (put-bytevector port (string->utf8 "This is unique."))))
+
+ (for-each (lambda (file)
+ (deduplicate file (sha256 data) #:store store))
+ identical)
+ (deduplicate unique (nar-sha256 unique) #:store store)
+
+ ;; (system (string-append "ls -lRia " store))
+ (cons* (apply = (map (compose stat:ino stat) identical))
+ (= (stat:ino (stat unique))
+ (stat:ino (stat (car identical))))
+ (stat:nlink (stat unique))
+ (map (compose stat:nlink stat) identical))))))
+
+(test-end "store-deduplication")
--
cgit 1.4.1