* String interpolation. Expressions like

    "--with-freetype2-library=" + freetype + "/lib"

  can now be written as

    "--with-freetype2-library=${freetype}/lib"

  An arbitrary expression can be enclosed within ${...}, not just
  identifiers.

* Escaping in string literals: \n, \r, \t interpreted as in C, any
  other character following \ is interpreted as-is.
  
* Newlines are now allowed in string literals.

7 files changed, 88 insertions, 15 deletions

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 1c2aafd91f..d70ac9f76a 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -231,7 +231,7 @@ static ATerm concatStrings(EvalState & state, const ATermVector & args)
 {
     ATermList context = ATempty;
     ostringstream s;
-    bool isPath;
+    bool isPath = false;
 
     for (ATermVector::const_iterator i = args.begin(); i != args.end(); ++i) {
         bool isPath2;
@@ -449,6 +449,14 @@ Expr evalExpr2(EvalState & state, Expr e)
         return makeList(ATconcat(l1, l2));
     }
 
+    /* String concatenation. */
+    ATermList es;
+    if (matchConcatStrings(e, es)) {
+        ATermVector args;
+        for (ATermIterator i(es); i; ++i) args.push_back(*i);
+        return concatStrings(state, args);
+    }
+
     /* Barf. */
     throw badTerm("invalid expression", e);
 }
diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l
index d5a14f517a..47f2bca1e6 100644
--- a/src/libexpr/lexer.l
+++ b/src/libexpr/lexer.l
@@ -3,6 +3,9 @@
 %option never-interactive
 
 
+%x STRING
+
+
 %{
 #include <string.h>
 #include <aterm2.h>
@@ -28,6 +31,9 @@ static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
     }
 }
 
+ATerm toATerm(const char * s);
+ATerm unescapeStr(const char * s);
+
 #define YY_USER_INIT initLoc(yylloc)
 #define YY_USER_ACTION adjustLoc(yylloc, yytext, yyleng);
 
@@ -36,7 +42,6 @@ static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
 
 ID          [a-zA-Z\_][a-zA-Z0-9\_\']*
 INT         [0-9]+
-STR         \"[^\n\"]*\"
 PATH        [a-zA-Z0-9\.\_\-\+]*(\/[a-zA-Z0-9\.\_\-\+]+)+
 URI         [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~\*\']+
 
@@ -61,19 +66,27 @@ inherit     { return INHERIT; }
 \/\/        { return UPDATE; }
 \+\+        { return CONCAT; }
 
-{ID}        { yylval->t = ATmake("<str>", yytext); return ID; /* !!! alloc */ }
+{ID}        { yylval->t = toATerm(yytext); return ID; /* !!! alloc */ }
 {INT}       { int n = atoi(yytext); /* !!! overflow */
               yylval->t = ATmake("<int>", n);
               return INT;
             }
-{STR}       { int len = strlen(yytext);
-              yytext[len - 1] = 0;
-              yylval->t = ATmake("<str>", yytext + 1);
-              yytext[len - 1] = '\"';
-              return STR; /* !!! alloc */
+
+\"          { BEGIN(STRING); return '"'; }
+<STRING>([^\$\"\\]|\\.|\$[^\{\$])+ {
+/* Note: a dollar *is* allowed as-is in a string, as long as it's
+   not followed by a open brace.  This should probably be disallowed
+   eventually. */
+              yylval->t = unescapeStr(yytext); /* !!! alloc */ 
+              return STR;
             }
-{PATH}      { yylval->t = ATmake("<str>", yytext); return PATH; /* !!! alloc */ }
-{URI}       { yylval->t = ATmake("<str>", yytext); return URI; /* !!! alloc */ }
+<STRING>\$\{  { BEGIN(INITIAL); return DOLLAR_CURLY; }
+<STRING>\"  { BEGIN(INITIAL); return '"'; }
+<STRING>.   return yytext[0]; /* just in case: shouldn't be reached */
+
+
+{PATH}      { yylval->t = toATerm(yytext); return PATH; /* !!! alloc */ }
+{URI}       { yylval->t = toATerm(yytext); return URI; /* !!! alloc */ }
 
 [ \t\n]+    /* eat up whitespace */
 \#[^\n]*    /* single-line comments */
@@ -83,3 +96,13 @@ inherit     { return INHERIT; }
 
 
 %%
+
+/* Horrible, disgusting hack: allow the parser to set the scanner
+   start condition back to STRING.  Necessary in interpolations like
+   "foo${expr}bar"; after the close brace we have to go back to the
+   STRING state. */
+void backToString(yyscan_t scanner)
+{
+    struct yyguts_t * yyg = (struct yyguts_t*) scanner;
+    BEGIN(STRING);
+}
diff --git a/src/libexpr/nixexpr-ast.def b/src/libexpr/nixexpr-ast.def
index fab560c99f..3f6473e940 100644
--- a/src/libexpr/nixexpr-ast.def
+++ b/src/libexpr/nixexpr-ast.def
@@ -19,6 +19,7 @@ SubPath | Expr Expr | Expr |
 OpHasAttr | Expr string | Expr |
 OpPlus | Expr Expr | Expr |
 OpConcat | Expr Expr | Expr |
+ConcatStrings | ATermList | Expr |
 Call | Expr Expr | Expr |
 Select | Expr string | Expr |
 Var | string | Expr |
diff --git a/src/libexpr/parser.cc b/src/libexpr/parser.cc
index 16b94fa62c..2ca1cab4af 100644
--- a/src/libexpr/parser.cc
+++ b/src/libexpr/parser.cc
@@ -71,9 +71,29 @@ const char * getPath(ParseData * data)
     return data->path.c_str();
 }
 
-int yyparse(yyscan_t scanner, ParseData * data);
+Expr unescapeStr(const char * s)
+{
+    string t;
+    char c;
+    while (c = *s++) {
+        if (c == '\\') {
+            assert(*s);
+            c = *s++;
+            if (c == 'n') t += "\n";
+            else if (c == 'r') t += "\r";
+            else if (c == 't') t += "\t";
+            else t += c;
+        }
+        else t += c;
+    }
+    return makeStr(toATerm(t));
 }
 
+int yyparse(yyscan_t scanner, ParseData * data);
+
+
+} /* end of C functions */
+
 
 static void checkAttrs(ATermMap & names, ATermList bnds)
 {
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index ec07a01919..cba390d8da 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -25,6 +25,7 @@ void parseError(void * data, char * error, int line, int column);
 ATerm absParsedPath(void * data, ATerm t);
 ATerm fixAttrs(int recursive, ATermList as);
 const char * getPath(void * data);
+void backToString(yyscan_t scanner);
 
 void yyerror(YYLTYPE * loc, yyscan_t scanner, void * data, char * s)
 {
@@ -73,9 +74,10 @@ static void freeAndUnprotect(void * p)
 
 %type <t> start expr expr_function expr_if expr_op
 %type <t> expr_app expr_select expr_simple bind inheritsrc formal
-%type <ts> binds ids expr_list formals
+%type <ts> binds ids expr_list formals string_parts
 %token <t> ID INT STR PATH URI
 %token IF THEN ELSE ASSERT WITH LET REC INHERIT EQ NEQ AND OR IMPL
+%token DOLLAR_CURLY /* == ${ */
 
 %nonassoc IMPL
 %left OR
@@ -142,7 +144,12 @@ expr_select
 expr_simple
   : ID { $$ = makeVar($1); }
   | INT { $$ = makeInt(ATgetInt((ATermInt) $1)); }
-  | STR { $$ = makeStr($1); }
+  | '"' string_parts '"' {
+      /* For efficiency, and to simplify parse trees a bit. */
+      if ($2 == ATempty) $$ = makeStr(toATerm(""));
+      else if (ATgetNext($2) == ATempty) $$ = ATgetFirst($2);
+      else $$ = makeConcatStrings(ATreverse($2));
+  }
   | PATH { $$ = makePath(absParsedPath(data, $1)); }
   | URI { $$ = makeUri($1); }
   | '(' expr ')' { $$ = $2; }
@@ -157,6 +164,12 @@ expr_simple
   | '[' expr_list ']' { $$ = makeList($2); }
   ;
 
+string_parts
+  : string_parts STR { $$ = ATinsert($1, $2); }
+  | string_parts DOLLAR_CURLY expr '}' { backToString(scanner); $$ = ATinsert($1, $3); }
+  | { $$ = ATempty; }
+  ;
+
 binds
   : binds bind { $$ = ATinsert($1, $2); }
   | { $$ = ATempty; }
diff --git a/tests/lang/eval-okay-string.exp b/tests/lang/eval-okay-string.exp
index 9c3f56c3f3..dd0e5e248d 100644
--- a/tests/lang/eval-okay-string.exp
+++ b/tests/lang/eval-okay-string.exp
@@ -1 +1 @@
-Str("foobar/a/b/c/d/foo/xyzzy/foo.txt/../foo/x/y")
+Str("foobar/a/b/c/d/foo/xyzzy/foo.txt/../foo/x/yescape: \"quote\" \n \\end\nof\nlinefoobarblaat")
diff --git a/tests/lang/eval-okay-string.nix b/tests/lang/eval-okay-string.nix
index b5280a0cd1..f2452e8574 100644
--- a/tests/lang/eval-okay-string.nix
+++ b/tests/lang/eval-okay-string.nix
@@ -1 +1,9 @@
-"foo" + "bar" + toString (/a/b + /c/d) + (/foo/bar + "/../xyzzy/." + "/foo.txt") + ("/../foo" + /x/y)
+"foo" + "bar"
+  + toString (/a/b + /c/d)
+  + (/foo/bar + "/../xyzzy/." + "/foo.txt")
+  + ("/../foo" + /x/y)
+  + "escape: \"quote\" \n \\"
+  + "end
+of
+line"
+  + "foo${if true then "b${"a" + "r"}" else "xyzzy"}blaat"