Add c++20 character types

eclipse-cdt · Mar 27, 2023 · d813935 · d813935
1 parent 2c30db5
commit d813935
Show file tree

Hide file tree

Showing 25 changed files with 158 additions and 9 deletions.
diff --git a/...rg.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2TestBase.java b/...rg.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2TestBase.java
@@ -157,6 +157,7 @@ private static Map<String, String> getStdMap() {
 	private static Map<String, String> getStdCpp20Map() {
 		Map<String, String> map = getStdMap();
 		map.put("__cpp_impl_three_way_comparison", "201907L");
+		map.put("__cpp_char8_t", "201811L");
 		return map;
 	}
 

diff --git a/...t.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/cxx20/CXX20CharacterTypes.java b/...t.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/cxx20/CXX20CharacterTypes.java
@@ -0,0 +1,46 @@
+/*******************************************************************************
+ * This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License 2.0
+ * which accompanies this distribution, and is available at
+ * https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *******************************************************************************/
+package org.eclipse.cdt.core.parser.tests.ast2.cxx20;
+
+import org.eclipse.cdt.core.dom.ast.IBasicType.Kind;
+import org.eclipse.cdt.core.dom.ast.IType;
+import org.eclipse.cdt.core.parser.ParserLanguage;
+import org.eclipse.cdt.core.parser.tests.ast2.AST2CPPTestBase;
+import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPBasicType;
+import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPPointerType;
+import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPQualifierType;
+
+import junit.framework.TestSuite;
+
+public class CXX20CharacterTypes extends AST2CPPTestBase {
+
+	public static TestSuite suite() {
+		return suite(CXX20CharacterTypes.class);
+	}
+
+	//	char  test_char;
+	//	char8_t  test_char8_t;
+	public void testCxx20CharacterTypes() throws Exception {
+		parseAndCheckBindings(ScannerKind.STDCPP20);
+	}
+
+	// auto u8 = u8"";
+	public void testStringLiteralPrefixChar8t() throws Exception {
+		IType eChar8 = createStringType(Kind.eChar8);
+
+		BindingAssertionHelper bh = getAssertionHelper(ParserLanguage.CPP, ScannerKind.STDCPP20);
+		assertType(bh.assertNonProblem("u8 = ", 2), eChar8);
+	}
+
+	protected IType createStringType(Kind kind) {
+		IType type = new CPPBasicType(kind, 0);
+		type = new CPPQualifierType(type, true, false);
+		return new CPPPointerType(type);
+	}
+}
diff --git a/core/org.eclipse.cdt.core/model/org/eclipse/cdt/internal/core/model/ASTStringUtil.java b/core/org.eclipse.cdt.core/model/org/eclipse/cdt/internal/core/model/ASTStringUtil.java
@@ -688,6 +688,9 @@ private static StringBuilder appendDeclSpecifierString(StringBuilder buffer, IAS
 			case IASTSimpleDeclSpecifier.t_wchar_t:
 				buffer.append(Keywords.WCHAR_T).append(' ');
 				break;
+			case IASTSimpleDeclSpecifier.t_char8_t:
+				buffer.append(Keywords.CHAR8_T).append(' ');
+				break;
 			case IASTSimpleDeclSpecifier.t_char16_t:
 				buffer.append(Keywords.CHAR16_T).append(' ');
 				break;

diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/ast/ASTSignatureUtil.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/ast/ASTSignatureUtil.java
@@ -733,6 +733,14 @@ public static String getSignature(IASTDeclSpecifier declSpec) {
 				result.append(Keywords.WCHAR_T);
 				needSpace = true;
 				break;
+			case IASTSimpleDeclSpecifier.t_char8_t:
+				if (needSpace) {
+					result.append(SPACE);
+					needSpace = false;
+				}
+				result.append(Keywords.CHAR8_T);
+				needSpace = true;
+				break;
 			case IASTSimpleDeclSpecifier.t_char16_t:
 				if (needSpace) {
 					result.append(SPACE);

diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/ast/ASTTypeUtil.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/ast/ASTTypeUtil.java
@@ -451,6 +451,11 @@ private static void appendTypeString(IType type, boolean normalize, StringBuilde
 					result.append(SPACE);
 				result.append(Keywords.WCHAR_T);
 				break;
+			case eChar8:
+				if (needSpace)
+					result.append(SPACE);
+				result.append(Keywords.CHAR8_T);
+				break;
 			case eChar16:
 				if (needSpace)
 					result.append(SPACE);

diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/ast/IASTSimpleDeclSpecifier.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/ast/IASTSimpleDeclSpecifier.java
@@ -140,6 +140,12 @@ public interface IASTSimpleDeclSpecifier extends IASTDeclSpecifier {
 	 */
 	public static final int t_decltype_auto = 18;
 
+	/**
+	 * <code>char8_t c;</code>
+	 * @since 8.1
+	 */
+	public static final int t_char8_t = 19;
+
 	/**
 	 * @since 5.1
 	 */

diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/ast/IBasicType.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/ast/IBasicType.java
@@ -37,7 +37,9 @@ enum Kind {
 		/** @since 5.10  */
 		eDecimal64,
 		/** @since 5.10  */
-		eDecimal128;
+		eDecimal128,
+		/** @since 8.1  */
+		eChar8;
 	}
 
 	/** @since 5.2 */

diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/ast/cpp/ICPPNodeFactory.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/ast/cpp/ICPPNodeFactory.java
@@ -258,6 +258,11 @@ public ICPPASTFoldExpression newFoldExpression(int opToken, boolean isComma, IAS
 	@Override
 	public ICPPASTLiteralExpression newLiteralExpression(int kind, String rep);
 
+	/**
+	 * @since 8.1
+	 */
+	public ICPPASTLiteralExpression newLiteralExpression(int kind, String rep, boolean useChar8Type);
+
 	/**
 	 * @since 6.5
 	 */

diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/IToken.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/IToken.java
@@ -119,6 +119,8 @@ public interface IToken {
 	int t_case = 62;
 	int t_catch = 63;
 	int t_char = 64;
+	/** @since 8.1 */
+	int t_char8_t = 8102;
 	/** @since 5.2 */
 	int t_char16_t = 5202;
 	/** @since 5.2 */

diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/Keywords.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/Keywords.java
@@ -49,6 +49,8 @@ public class Keywords {
 	public static final String CASE = "case";
 	public static final String CATCH = "catch";
 	public static final String CHAR = "char";
+	/** @since 8.1 */
+	public static final String CHAR8_T = "char8_t";
 	/** @since 5.2 */
 	public static final String CHAR16_T = "char16_t";
 	/** @since 5.2 */
@@ -157,6 +159,8 @@ public class Keywords {
 	public static final char[] cCASE = "case".toCharArray();
 	public static final char[] cCATCH = "catch".toCharArray();
 	public static final char[] cCHAR = "char".toCharArray();
+	/** @since 8.1 */
+	public static final char[] cCHAR8_T = CHAR8_T.toCharArray();
 	/** @since 5.2 */
 	public static final char[] cCHAR16_T = CHAR16_T.toCharArray();
 	/** @since 5.2 */
@@ -395,6 +399,10 @@ private static void addC(CharArrayIntMap ckeywords) {
 		ckeywords.put(Keywords.c_IMAGINARY, IToken.t__Imaginary);
 	}
 
+	private static void addCpp20(CharArrayIntMap cppkeywords) {
+		cppkeywords.put(Keywords.cCHAR8_T, IToken.t_char8_t);
+	}
+
 	private static void addCpp(CharArrayIntMap cppkeywords) {
 		cppkeywords.put(Keywords.cALIGNAS, IToken.t_alignas);
 		cppkeywords.put(Keywords.cALIGNOF, IToken.t_alignof);

diff --git a/...cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/AbstractGNUSourceCodeParser.java b/...cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/AbstractGNUSourceCodeParser.java
@@ -2778,6 +2778,7 @@ protected boolean canBeTypeSpecifier() throws EndOfFileException {
 		case IToken.tCOLONCOLON:
 		case IToken.t_void:
 		case IToken.t_char:
+		case IToken.t_char8_t:
 		case IToken.t_char16_t:
 		case IToken.t_char32_t:
 		case IToken.t_wchar_t:

diff --git a/...clipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/ArithmeticConversion.java b/...clipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/ArithmeticConversion.java
@@ -136,6 +136,7 @@ private boolean isIntegralOrUnscopedEnum(IType op1) {
 			switch (kind) {
 			case eBoolean:
 			case eChar:
+			case eChar8:
 			case eChar16:
 			case eChar32:
 			case eInt:
@@ -240,6 +241,7 @@ private IBasicType promote(IType type, Domain domain) {
 			switch (kind) {
 			case eBoolean:
 			case eChar:
+			case eChar8:
 			case eWChar:
 			case eChar16:
 				return createBasicType(Kind.eInt, domain.getModifier());
@@ -384,6 +386,7 @@ public static boolean fitsIntoType(IBasicType basicTarget, long n) {
 			return n < (Integer.MAX_VALUE + 1L) * 2;
 
 		case eChar:
+		case eChar8:
 			return 0 <= n && n <= 0xFF;
 
 		case eChar16:
@@ -415,6 +418,7 @@ public static boolean fitsIntoType(IBasicType basicTarget, long n) {
 	private static long getApproximateSize(IBasicType type) {
 		switch (type.getKind()) {
 		case eChar:
+		case eChar8:
 			return 1;
 		case eWChar:
 			return 2;

diff --git a/...rg.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/SizeofCalculator.java b/...rg.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/SizeofCalculator.java
@@ -230,6 +230,7 @@ private SizeAndAlignment sizeAndAlignment(IBasicType type) {
 		case eBoolean:
 			return sizeof_bool;
 		case eChar:
+		case eChar8:
 			return SIZE_1;
 		case eInt:
 			return type.isShort() ? sizeof_short

diff --git a/...cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/CPPASTLiteralExpression.java b/...cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/CPPASTLiteralExpression.java
@@ -84,17 +84,23 @@ public class CPPASTLiteralExpression extends ASTNode implements ICPPASTLiteralEx
 	private ICPPEvaluation fEvaluation;
 	private IBinding fUserDefinedLiteralOperator;
 	private IASTImplicitName[] fImplicitNames;
+	private final boolean fUseChar8Type;
 
 	public CPPASTLiteralExpression(int kind, char[] value) {
-		this(kind, value, CharArrayUtils.EMPTY);
+		this(kind, value, CharArrayUtils.EMPTY, false);
 	}
 
-	public CPPASTLiteralExpression(int kind, char[] value, char[] numericCompilerSuffixes) {
+	public CPPASTLiteralExpression(int kind, char[] value, boolean useChar8Type) {
+		this(kind, value, CharArrayUtils.EMPTY, useChar8Type);
+	}
+
+	public CPPASTLiteralExpression(int kind, char[] value, char[] numericCompilerSuffixes, boolean useChar8Type) {
 		fKind = kind;
 		fSuffix = getSuffix(kind, value, CharArrayUtils.EMPTY);
 		fLiteral = getLiteral(value, fSuffix);
 		fNumericCompilerSuffixes = (numericCompilerSuffixes == null) ? CharArrayUtils.EMPTY : numericCompilerSuffixes;
 		fStringLiteralSize = -1;
+		fUseChar8Type = useChar8Type;
 	}
 
 	private CPPASTLiteralExpression(CPPASTLiteralExpression other) {
@@ -106,6 +112,7 @@ private CPPASTLiteralExpression(CPPASTLiteralExpression other) {
 		fEvaluation = other.fEvaluation;
 		fUserDefinedLiteralOperator = other.fUserDefinedLiteralOperator;
 		fImplicitNames = other.fImplicitNames;
+		fUseChar8Type = other.fUseChar8Type;
 	}
 
 	@Override
@@ -437,7 +444,14 @@ public Kind getBasicCharKind() {
 			return Kind.eChar32;
 		case 'u':
 			// Bug 526724 u8 should result in Kind.eChar
-			if (fLiteral[1] != '8') {
+			if (fLiteral[1] == '8') {
+				boolean doBreakpoint = true;
+				if (fUseChar8Type) {
+					return Kind.eChar8;
+				} else {
+					return Kind.eChar;
+				}
+			} else {
 				return Kind.eChar16;
 			}
 			//$FALL-THROUGH$

diff --git a/...t.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/CPPASTSimpleDeclSpecifier.java b/...t.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/CPPASTSimpleDeclSpecifier.java
@@ -85,6 +85,8 @@ private int getType(Kind kind) {
 			return t_char;
 		case eWChar:
 			return t_wchar_t;
+		case eChar8:
+			return t_char8_t;
 		case eChar16:
 			return t_char16_t;
 		case eChar32:

diff --git a/...rg.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/CPPBasicType.java b/...rg.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/CPPBasicType.java
@@ -127,6 +127,8 @@ static Kind getKind(final int simpleDeclSpecType) {
 			return Kind.eChar;
 		case IASTSimpleDeclSpecifier.t_wchar_t:
 			return Kind.eWChar;
+		case IASTSimpleDeclSpecifier.t_char8_t:
+			return Kind.eChar8;
 		case IASTSimpleDeclSpecifier.t_char16_t:
 			return Kind.eChar16;
 		case IASTSimpleDeclSpecifier.t_char32_t:
@@ -342,6 +344,7 @@ public int getType() {
 		case eBoolean:
 			return t_bool;
 		case eChar:
+		case eChar8:
 		case eChar16:
 		case eChar32:
 			return t_char;

diff --git a/....eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/CPPNodeFactory.java b/....eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/CPPNodeFactory.java
@@ -584,12 +584,17 @@ public ICPPASTLinkageSpecification newLinkageSpecification(String literal) {
 
 	@Override
 	public ICPPASTLiteralExpression newLiteralExpression(int kind, String rep) {
-		return new CPPASTLiteralExpression(kind, rep.toCharArray());
+		return new CPPASTLiteralExpression(kind, rep.toCharArray(), false);
+	}
+
+	@Override
+	public ICPPASTLiteralExpression newLiteralExpression(int kind, String rep, boolean useChar8Type) {
+		return new CPPASTLiteralExpression(kind, rep.toCharArray(), useChar8Type);
 	}
 
 	@Override
 	public ICPPASTLiteralExpression newLiteralExpression(int kind, String rep, char[] numericCompilerSuffixes) {
-		return new CPPASTLiteralExpression(kind, rep.toCharArray(), numericCompilerSuffixes);
+		return new CPPASTLiteralExpression(kind, rep.toCharArray(), numericCompilerSuffixes, false);
 	}
 
 	@Override

diff --git a/...ipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/GNUCPPSourceParser.java b/...ipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/GNUCPPSourceParser.java
@@ -209,6 +209,7 @@ protected static enum DtorStrategy {
 	private final boolean supportUserDefinedLiterals;
 	private final boolean supportGCCStyleDesignators;
 	private final boolean supportFoldExpression;
+	private final boolean supportChar8TypeLiterals;
 
 	private final IIndex index;
 	protected ICPPASTTranslationUnit translationUnit;
@@ -247,6 +248,7 @@ public GNUCPPSourceParser(IScanner scanner, ParserMode mode, IParserLogService l
 		fContextSensitiveTokens = createContextSensitiveTokenMap(config);
 		additionalNumericalSuffixes = scanner.getAdditionalNumericLiteralSuffixes();
 		supportFoldExpression = true;
+		supportChar8TypeLiterals = scanner.getMacroDefinitions().containsKey("__cpp_char8_t"); //$NON-NLS-1$
 	}
 
 	@Override
@@ -672,6 +674,7 @@ private int endsTemplateIDInBinaryExpression() {
 		// Start of a postfix expression
 		case IToken.t_typename:
 		case IToken.t_char:
+		case IToken.t_char8_t:
 		case IToken.t_char16_t:
 		case IToken.t_char32_t:
 		case IToken.t_wchar_t:
@@ -2060,6 +2063,7 @@ private IASTExpression postfixExpression(CastExprCtx ctx, ITemplateIdStrategy st
 		// simple-type-specifier braced-init-list
 		case IToken.t_typename:
 		case IToken.t_char:
+		case IToken.t_char8_t:
 		case IToken.t_char16_t:
 		case IToken.t_char32_t:
 		case IToken.t_wchar_t:
@@ -2253,7 +2257,8 @@ protected IASTExpression primaryExpression(CastExprCtx ctx, ITemplateIdStrategy
 		case IToken.tUTF32CHAR:
 		case IToken.tUSER_DEFINED_CHAR_LITERAL:
 			t = consume();
-			literalExpr = getNodeFactory().newLiteralExpression(IASTLiteralExpression.lk_char_constant, t.getImage());
+			literalExpr = getNodeFactory().newLiteralExpression(IASTLiteralExpression.lk_char_constant, t.getImage(),
+					supportChar8TypeLiterals);
 			literalExprWithRange = setRange(literalExpr, t.getOffset(), t.getEndOffset());
 			break;
 		case IToken.t_false:
@@ -2349,7 +2354,7 @@ private ICPPASTLiteralExpression stringLiteral() throws EndOfFileException, Back
 		}
 		IToken t = consume();
 		ICPPASTLiteralExpression r = getNodeFactory().newLiteralExpression(IASTLiteralExpression.lk_string_literal,
-				t.getImage());
+				t.getImage(), supportChar8TypeLiterals);
 		return setRange(r, t.getOffset(), t.getEndOffset());
 	}
 
@@ -3726,6 +3731,13 @@ private Decl declSpecifierSeq(final DeclarationOptions option, final boolean sin
 					encounteredRawType = true;
 					endOffset = consume().getEndOffset();
 					break;
+				case IToken.t_char8_t:
+					if (encounteredTypename)
+						break declSpecifiers;
+					simpleType = IASTSimpleDeclSpecifier.t_char8_t;
+					encounteredRawType = true;
+					endOffset = consume().getEndOffset();
+					break;
 				case IToken.t_char16_t:
 					if (encounteredTypename)
 						break declSpecifiers;

diff --git a/...arser/org/eclipse/cdt/internal/core/dom/parser/cpp/semantics/AggregateInitialization.java b/...arser/org/eclipse/cdt/internal/core/dom/parser/cpp/semantics/AggregateInitialization.java
@@ -261,7 +261,7 @@ private static boolean isCharArray(IType target) {
 		ICPPBasicType t = getBasicTypeFromArray(target);
 		if (t != null) {
 			Kind k = t.getKind();
-			return k == Kind.eChar || k == Kind.eChar16 || k == Kind.eChar32 || k == Kind.eWChar;
+			return k == Kind.eChar || k == Kind.eChar8 || k == Kind.eChar16 || k == Kind.eChar32 || k == Kind.eWChar;
 		}
 		return false;
 	}