feat: Add a syntax for describing label literals (#5131)

* feat: Add a syntax for describing label literals

Adds the syntax and locks it behind the `LabelPolymorphism` feature flag.
(#4928 exists as the next step to allow users to actually opt-in to it).

Closes #4927

* test: Allow label literals in testcases

* chore: make generate

ast/ast.go

@@ -123,6 +123,7 @@ func (*IntegerLiteral) node()         {}
 func (*PipeLiteral) node()            {}
 func (*RegexpLiteral) node()          {}
 func (*StringLiteral) node()          {}
+func (*LabelLiteral) node()           {}
 func (*UnsignedIntegerLiteral) node() {}
 
 func (*NamedType) node()      {}
@@ -885,6 +886,7 @@ func (*PipeExpression) expression()         {}
 func (*PipeLiteral) expression()            {}
 func (*RegexpLiteral) expression()          {}
 func (*StringLiteral) expression()          {}
+func (*LabelLiteral) expression()           {}
 func (*UnaryExpression) expression()        {}
 func (*UnsignedIntegerLiteral) expression() {}
 
@@ -1581,6 +1583,30 @@ func (l *StringLiteral) Copy() Node {
 	return nl
 }
 
+// LabelLiteral expressions begin and end with double quote marks.
+type LabelLiteral struct {
+	BaseNode
+	// Value is the unescaped value of the string literal
+	Value string `json:"value"`
+}
+
+// LabelLiterals are valid object keys
+func (l *LabelLiteral) Key() string {
+	return l.Value
+}
+
+func (*LabelLiteral) Type() string { return "LabelLiteral" }
+
+func (l *LabelLiteral) Copy() Node {
+	if l == nil {
+		return l
+	}
+	nl := new(LabelLiteral)
+	*nl = *l
+	nl.BaseNode = l.BaseNode.Copy()
+	return nl
+}
+
 // BooleanLiteral represent boolean values
 type BooleanLiteral struct {
 	BaseNode

ast/asttest/cmpopts.go

@@ -32,6 +32,7 @@ var IgnoreBaseNodeOptions = []cmp.Option{
 	cmpopts.IgnoreFields(ast.IndexExpression{}, "BaseNode"),
 	cmpopts.IgnoreFields(ast.IntegerLiteral{}, "BaseNode"),
 	cmpopts.IgnoreFields(ast.InterpolatedPart{}, "BaseNode"),
+	cmpopts.IgnoreFields(ast.LabelLiteral{}, "BaseNode"),
 	cmpopts.IgnoreFields(ast.LogicalExpression{}, "BaseNode"),
 	cmpopts.IgnoreFields(ast.MemberAssignment{}, "BaseNode"),
 	cmpopts.IgnoreFields(ast.MemberExpression{}, "BaseNode"),

ast/json.go

@@ -990,6 +990,17 @@ func (l *StringLiteral) MarshalJSON() ([]byte, error) {
 	}
 	return json.Marshal(raw)
 }
+func (l *LabelLiteral) MarshalJSON() ([]byte, error) {
+	type Alias LabelLiteral
+	raw := struct {
+		Type string `json:"type"`
+		*Alias
+	}{
+		Type:  l.Type(),
+		Alias: (*Alias)(l),
+	}
+	return json.Marshal(raw)
+}
 func (l *BooleanLiteral) MarshalJSON() ([]byte, error) {
 	type Alias BooleanLiteral
 	raw := struct {
@@ -1685,6 +1696,8 @@ func unmarshalNode(msg json.RawMessage) (Node, error) {
 		node = new(ParenExpression)
 	case "StringLiteral":
 		node = new(StringLiteral)
+	case "LabelLiteral":
+		node = new(LabelLiteral)
 	case "BooleanLiteral":
 		node = new(BooleanLiteral)
 	case "FloatLiteral":

docs/SPEC.md

@@ -365,6 +365,18 @@ Examples:
 
 The regular expression syntax is defined by [RE2](https://github.com/google/re2/wiki/Syntax).
 
+#### Label literals
+
+```flux
+
+.mylabel
+._value
+."with spaces"
+```
+
+A label literal represents a "label" used to refer to specific record fields. They have two variants, where the `.` can
+be followed by either an identifier or a string literal (allowing labels with characters that are not allowed in identifiers to be specified).
+
 ### Variables
 
 A variable represents a storage location for a single value.

execute/executetest/dependencies.go

@@ -30,6 +30,7 @@ var testFlags = map[string]interface{}{
 	// "aggregateTransformationTransport": true,
 	// "groupTransformationGroup":         true,
 	// "optimizeUnionTransformation": true,
+	"labelPolymorphism":         true,
 	"vectorizedMap":             true,
 	"vectorizedConst":           true,
 	"vectorizedConditionals":    true,

libflux/flux-core/src/ast/mod.rs

@@ -189,6 +189,8 @@ pub enum Expression {
     Regexp(RegexpLit),
     #[serde(rename = "PipeLiteral")]
     PipeLit(PipeLit),
+    #[serde(rename = "LabelLiteral")]
+    Label(LabelLit),
 
     #[serde(rename = "BadExpression")]
     Bad(Box<BadExpr>),
@@ -220,6 +222,7 @@ impl Expression {
             Expression::DateTime(wrapped) => &wrapped.base,
             Expression::Regexp(wrapped) => &wrapped.base,
             Expression::PipeLit(wrapped) => &wrapped.base,
+            Expression::Label(wrapped) => &wrapped.base,
             Expression::Bad(wrapped) => &wrapped.base,
             Expression::StringExpr(wrapped) => &wrapped.base,
             Expression::Paren(wrapped) => &wrapped.base,
@@ -614,7 +617,7 @@ pub enum MonoType {
     #[serde(rename = "FunctionType")]
     Function(Box<FunctionType>),
     #[serde(rename = "LabelType")]
-    Label(Box<StringLit>),
+    Label(Box<LabelLit>),
 }
 
 impl MonoType {
@@ -727,7 +730,7 @@ pub enum ParameterType {
         monotype: MonoType,
         // Default value for this parameter. Currently only string literals are supported
         // (to allow default labels to be specified)
-        default: Option<StringLit>,
+        default: Option<LabelLit>,
     },
     Pipe {
         #[serde(skip_serializing_if = "BaseNode::is_empty")]
@@ -1449,6 +1452,18 @@ pub struct UintLit {
     pub value: u64,
 }
 
+/// LabelLit represents a label. Used to specify specific record fields.
+#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)]
+#[allow(missing_docs)]
+pub struct LabelLit {
+    #[serde(skip_serializing_if = "BaseNode::is_empty")]
+    #[serde(default)]
+    #[serde(flatten)]
+    pub base: BaseNode,
+
+    pub value: String,
+}
+
 struct U64Visitor;
 
 impl<'de> Visitor<'de> for U64Visitor {

libflux/flux-core/src/ast/walk/mod.rs

@@ -71,6 +71,8 @@ pub enum Node<'a> {
     RegexpLit(&'a RegexpLit),
     #[display(fmt = "PipeLit")]
     PipeLit(&'a PipeLit),
+    #[display(fmt = "LabelLit")]
+    LabelLit(&'a LabelLit),
 
     #[display(fmt = "BadExpr")]
     BadExpr(&'a BadExpr),
@@ -153,6 +155,7 @@ impl<'a> Node<'a> {
             Node::DateTimeLit(n) => &n.base,
             Node::RegexpLit(n) => &n.base,
             Node::PipeLit(n) => &n.base,
+            Node::LabelLit(n) => &n.base,
             Node::BadExpr(n) => &n.base,
             Node::ExprStmt(n) => &n.base,
             Node::OptionStmt(n) => &n.base,
@@ -203,6 +206,7 @@ impl<'a> Node<'a> {
             Expression::DateTime(e) => Node::DateTimeLit(e),
             Expression::Regexp(e) => Node::RegexpLit(e),
             Expression::PipeLit(e) => Node::PipeLit(e),
+            Expression::Label(e) => Node::LabelLit(e),
             Expression::Bad(e) => Node::BadExpr(e),
         }
     }
@@ -377,6 +381,7 @@ where
             Node::DateTimeLit(_) => {}
             Node::RegexpLit(_) => {}
             Node::PipeLit(_) => {}
+            Node::LabelLit(_) => {}
             Node::BadExpr(n) => {
                 if let Some(e) = &n.expression {
                     walk(v, Node::from_expr(e));
@@ -455,7 +460,7 @@ where
 
                     walk(v, Node::MonoType(&f.monotype));
                 }
-                MonoType::Label(lit) => walk(v, Node::StringLit(lit)),
+                MonoType::Label(_) => (),
             },
             Node::PropertyType(n) => {
                 walk(v, Node::from_property_key(&n.name));
@@ -475,7 +480,7 @@ where
                     walk(v, Node::Identifier(name));
                     walk(v, Node::MonoType(monotype));
                     if let Some(default) = default {
-                        walk(v, Node::StringLit(default));
+                        walk(v, Node::LabelLit(default));
                     }
                 }
                 ParameterType::Pipe { name, monotype, .. } => {

libflux/flux-core/src/formatter/mod.rs

@@ -375,6 +375,7 @@ impl<'doc> Formatter<'doc> {
             Node::DateTimeLit(x) => self.format_date_time_literal(x),
             Node::RegexpLit(x) => self.format_regexp_literal(x),
             Node::PipeLit(x) => self.format_pipe_literal(x),
+            Node::LabelLit(x) => self.format_label_literal(x),
             Node::ExprStmt(x) => self.format_expression_statement(x),
             Node::OptionStmt(x) => self.format_option_statement(x),
             Node::ReturnStmt(x) => self.format_return_statement(x),
@@ -601,7 +602,12 @@ impl<'doc> Formatter<'doc> {
                     self.format_monotype(&n.monotype),
                 ]
             }
-            ast::MonoType::Label(label) => self.format_string_literal(label),
+            ast::MonoType::Label(label) => docs![
+                arena,
+                self.format_comments(&label.base.comments),
+                ".",
+                &label.value,
+            ],
         }
         .group()
     }
@@ -645,7 +651,7 @@ impl<'doc> Formatter<'doc> {
                     ": ",
                     self.format_monotype(monotype),
                     match default {
-                        Some(default) => docs![arena, " = ", self.format_string_literal(default)],
+                        Some(default) => docs![arena, " = ", self.format_label_literal(default)],
                         None => arena.nil(),
                     }
                 ]
@@ -1372,6 +1378,7 @@ impl<'doc> Formatter<'doc> {
             ast::Expression::DateTime(expr) => self.format_date_time_literal(expr),
             ast::Expression::Regexp(expr) => self.format_regexp_literal(expr),
             ast::Expression::PipeLit(expr) => self.format_pipe_literal(expr),
+            ast::Expression::Label(expr) => self.format_label_literal(expr),
             ast::Expression::Bad(expr) => {
                 self.err = Some(anyhow!("bad expression"));
                 arena.nil()
@@ -1612,6 +1619,12 @@ impl<'doc> Formatter<'doc> {
         docs![arena, self.format_comments(&n.base.comments), "<-"]
     }
 
+    fn format_label_literal(&mut self, n: &'doc ast::LabelLit) -> Doc<'doc> {
+        let arena = self.arena;
+
+        docs![arena, self.format_comments(&n.base.comments), ".", &n.value]
+    }
+
     fn format_text_part(&mut self, n: &'doc ast::TextPart) -> Doc<'doc> {
         let arena = self.arena;
 
@@ -2010,6 +2023,7 @@ fn starts_with_comment(n: Node) -> bool {
         Node::DateTimeLit(n) => !n.base.comments.is_empty(),
         Node::RegexpLit(n) => !n.base.comments.is_empty(),
         Node::PipeLit(n) => !n.base.comments.is_empty(),
+        Node::LabelLit(n) => !n.base.comments.is_empty(),
         Node::BadExpr(_) => false,
         Node::ExprStmt(n) => starts_with_comment(Node::from_expr(&n.expression)),
         Node::OptionStmt(n) => !n.base.comments.is_empty(),

libflux/flux-core/src/parser/mod.rs

@@ -562,6 +562,7 @@ impl<'input> Parser<'input> {
             }
             TokenType::LBrace => self.parse_record_type(),
             TokenType::LParen => self.parse_function_type(),
+            TokenType::Dot => MonoType::Label(Box::new(self.parse_label_literal())),
             TokenType::Ident if t.lit == "stream" => {
                 let start = self.expect(TokenType::Ident);
                 self.open(TokenType::LBrack, TokenType::RBrack);
@@ -656,7 +657,7 @@ impl<'input> Parser<'input> {
 
                 let default = if self.peek().tok == TokenType::Assign {
                     self.expect(TokenType::Assign);
-                    Some(self.parse_string_literal())
+                    Some(self.parse_label_literal())
                 } else {
                     None
                 };
@@ -1469,6 +1470,7 @@ impl<'input> Parser<'input> {
             }
             TokenType::LBrace => Expression::Object(Box::new(self.parse_object_literal())),
             TokenType::LParen => self.parse_paren_expression(),
+            TokenType::Dot => Expression::Label(self.parse_label_literal()),
             // We got a bad token, do not consume it, but use it in the message.
             // Other methods will match BadExpr and consume the token if needed.
             _ => {
@@ -1611,6 +1613,20 @@ impl<'input> Parser<'input> {
             }
         }
     }
+
+    fn parse_label_literal(&mut self) -> LabelLit {
+        let dot = self.expect(TokenType::Dot);
+        let tok = self.expect_one_of(&[TokenType::Ident, TokenType::String]);
+
+        let base = self.base_node_from_tokens(&dot, &tok);
+        let value = match tok.tok {
+            TokenType::String => self.new_string_literal(tok).value,
+            _ => tok.lit,
+        };
+
+        LabelLit { base, value }
+    }
+
     fn parse_regexp_literal(&mut self) -> RegexpLit {
         let t = self.expect(TokenType::Regex);
         let value = strconv::parse_regex(t.lit.as_str());