Add c++20 character types

core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2CPPTestBase.java

@@ -26,15 +26,27 @@ public AST2CPPTestBase(String name) {
 	}
 
 	protected IASTTranslationUnit parseAndCheckBindings(String code) throws Exception {
-		return parseAndCheckBindings(code, CPP);
+		return parseAndCheckBindings(code, ScannerKind.STD);
+	}
+
+	protected IASTTranslationUnit parseAndCheckBindings(String code, ScannerKind scannerKind) throws Exception {
+		return parseAndCheckBindings(code, CPP, scannerKind);
 	}
 
 	protected IASTTranslationUnit parseAndCheckBindings() throws Exception {
+		return parseAndCheckBindings(ScannerKind.STD);
+	}
+
+	protected IASTTranslationUnit parseAndCheckBindings(ScannerKind scannerKind) throws Exception {
 		String code = getAboveComment();
-		return parseAndCheckBindings(code);
+		return parseAndCheckBindings(code, scannerKind);
 	}
 
 	protected BindingAssertionHelper getAssertionHelper() throws ParserException, IOException {
-		return getAssertionHelper(CPP);
+		return getAssertionHelper(ScannerKind.GNU);
+	}
+
+	protected BindingAssertionHelper getAssertionHelper(ScannerKind scannerKind) throws ParserException, IOException {
+		return getAssertionHelper(CPP, scannerKind);
 	}
 }

core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2TestBase.java

@@ -157,6 +157,7 @@ private static Map<String, String> getStdMap() {
 	private static Map<String, String> getStdCpp20Map() {
 		Map<String, String> map = getStdMap();
 		map.put("__cpp_impl_three_way_comparison", "201907L");
+		map.put("__cpp_char8_t", "201811L");
 		return map;
 	}
 

core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/cxx20/CXX20CharacterTypes.java

@@ -0,0 +1,46 @@
+/*******************************************************************************
+ * This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License 2.0
+ * which accompanies this distribution, and is available at
+ * https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *******************************************************************************/
+package org.eclipse.cdt.core.parser.tests.ast2.cxx20;
+
+import org.eclipse.cdt.core.dom.ast.IBasicType.Kind;
+import org.eclipse.cdt.core.dom.ast.IType;
+import org.eclipse.cdt.core.parser.ParserLanguage;
+import org.eclipse.cdt.core.parser.tests.ast2.AST2CPPTestBase;
+import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPBasicType;
+import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPPointerType;
+import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPQualifierType;
+
+import junit.framework.TestSuite;
+
+public class CXX20CharacterTypes extends AST2CPPTestBase {
+
+	public static TestSuite suite() {
+		return suite(CXX20CharacterTypes.class);
+	}
+
+	//	char  test_char;
+	//	char8_t  test_char8_t;
+	public void testCxx20CharacterTypes() throws Exception {
+		parseAndCheckBindings(ScannerKind.STDCPP20);
+	}
+
+	// auto u8 = u8"";
+	public void testStringLiteralPrefixChar8t() throws Exception {
+		IType eChar8 = createStringType(Kind.eChar8);
+
+		BindingAssertionHelper bh = getAssertionHelper(ParserLanguage.CPP, ScannerKind.STDCPP20);
+		assertType(bh.assertNonProblem("u8 = ", 2), eChar8);
+	}
+
+	protected IType createStringType(Kind kind) {
+		IType type = new CPPBasicType(kind, 0);
+		type = new CPPQualifierType(type, true, false);
+		return new CPPPointerType(type);
+	}
+}

core/org.eclipse.cdt.core/model/org/eclipse/cdt/internal/core/model/ASTStringUtil.java

@@ -688,6 +688,9 @@ private static StringBuilder appendDeclSpecifierString(StringBuilder buffer, IAS
 			case IASTSimpleDeclSpecifier.t_wchar_t:
 				buffer.append(Keywords.WCHAR_T).append(' ');
 				break;
+			case IASTSimpleDeclSpecifier.t_char8_t:
+				buffer.append(Keywords.CHAR8_T).append(' ');
+				break;
 			case IASTSimpleDeclSpecifier.t_char16_t:
 				buffer.append(Keywords.CHAR16_T).append(' ');
 				break;

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/ast/ASTSignatureUtil.java

@@ -733,6 +733,14 @@ public static String getSignature(IASTDeclSpecifier declSpec) {
 				result.append(Keywords.WCHAR_T);
 				needSpace = true;
 				break;
+			case IASTSimpleDeclSpecifier.t_char8_t:
+				if (needSpace) {
+					result.append(SPACE);
+					needSpace = false;
+				}
+				result.append(Keywords.CHAR8_T);
+				needSpace = true;
+				break;
 			case IASTSimpleDeclSpecifier.t_char16_t:
 				if (needSpace) {
 					result.append(SPACE);

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/ast/ASTTypeUtil.java

@@ -451,6 +451,11 @@ private static void appendTypeString(IType type, boolean normalize, StringBuilde
 					result.append(SPACE);
 				result.append(Keywords.WCHAR_T);
 				break;
+			case eChar8:
+				if (needSpace)
+					result.append(SPACE);
+				result.append(Keywords.CHAR8_T);
+				break;
 			case eChar16:
 				if (needSpace)
 					result.append(SPACE);

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/ast/IASTSimpleDeclSpecifier.java

@@ -140,6 +140,12 @@ public interface IASTSimpleDeclSpecifier extends IASTDeclSpecifier {
 	 */
 	public static final int t_decltype_auto = 18;
 
+	/**
+	 * <code>char8_t c;</code>
+	 * @since 8.1
+	 */
+	public static final int t_char8_t = 19;
+
 	/**
 	 * @since 5.1
 	 */

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/ast/IBasicType.java

@@ -37,7 +37,9 @@ enum Kind {
 		/** @since 5.10  */
 		eDecimal64,
 		/** @since 5.10  */
-		eDecimal128;
+		eDecimal128,
+		/** @since 8.1 */
+		eChar8;
 	}
 
 	/** @since 5.2 */

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/ast/cpp/ICPPNodeFactory.java

@@ -258,6 +258,11 @@ public ICPPASTFoldExpression newFoldExpression(int opToken, boolean isComma, IAS
 	@Override
 	public ICPPASTLiteralExpression newLiteralExpression(int kind, String rep);
 
+	/**
+	 * @since 8.1
+	 */
+	public ICPPASTLiteralExpression newLiteralExpression(int kind, String rep, boolean useChar8Type);
+
 	/**
 	 * @since 6.5
 	 */

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/IToken.java

@@ -119,6 +119,8 @@ public interface IToken {
 	int t_case = 62;
 	int t_catch = 63;
 	int t_char = 64;
+	/** @since 8.1 */
+	int t_char8_t = 8102;
 	/** @since 5.2 */
 	int t_char16_t = 5202;
 	/** @since 5.2 */

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/Keywords.java

@@ -49,6 +49,8 @@ public class Keywords {
 	public static final String CASE = "case";
 	public static final String CATCH = "catch";
 	public static final String CHAR = "char";
+	/** @since 8.1 */
+	public static final String CHAR8_T = "char8_t";
 	/** @since 5.2 */
 	public static final String CHAR16_T = "char16_t";
 	/** @since 5.2 */
@@ -157,6 +159,8 @@ public class Keywords {
 	public static final char[] cCASE = "case".toCharArray();
 	public static final char[] cCATCH = "catch".toCharArray();
 	public static final char[] cCHAR = "char".toCharArray();
+	/** @since 8.1 */
+	public static final char[] cCHAR8_T = CHAR8_T.toCharArray();
 	/** @since 5.2 */
 	public static final char[] cCHAR16_T = CHAR16_T.toCharArray();
 	/** @since 5.2 */
@@ -395,6 +399,10 @@ private static void addC(CharArrayIntMap ckeywords) {
 		ckeywords.put(Keywords.c_IMAGINARY, IToken.t__Imaginary);
 	}
 
+	private static void addCpp20(CharArrayIntMap cppkeywords) {
+		cppkeywords.put(Keywords.cCHAR8_T, IToken.t_char8_t);
+	}
+
 	private static void addCpp(CharArrayIntMap cppkeywords) {
 		cppkeywords.put(Keywords.cALIGNAS, IToken.t_alignas);
 		cppkeywords.put(Keywords.cALIGNOF, IToken.t_alignof);

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/AbstractGNUSourceCodeParser.java

@@ -2787,6 +2787,7 @@ protected boolean canBeTypeSpecifier() throws EndOfFileException {
 		case IToken.tCOLONCOLON:
 		case IToken.t_void:
 		case IToken.t_char:
+		case IToken.t_char8_t:
 		case IToken.t_char16_t:
 		case IToken.t_char32_t:
 		case IToken.t_wchar_t:

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/ArithmeticConversion.java

@@ -136,6 +136,7 @@ private boolean isIntegralOrUnscopedEnum(IType op1) {
 			switch (kind) {
 			case eBoolean:
 			case eChar:
+			case eChar8:
 			case eChar16:
 			case eChar32:
 			case eInt:
@@ -240,6 +241,7 @@ private IBasicType promote(IType type, Domain domain) {
 			switch (kind) {
 			case eBoolean:
 			case eChar:
+			case eChar8:
 			case eWChar:
 			case eChar16:
 				return createBasicType(Kind.eInt, domain.getModifier());
@@ -384,6 +386,7 @@ public static boolean fitsIntoType(IBasicType basicTarget, long n) {
 			return n < (Integer.MAX_VALUE + 1L) * 2;
 
 		case eChar:
+		case eChar8:
 			return 0 <= n && n <= 0xFF;
 
 		case eChar16:
@@ -415,6 +418,7 @@ public static boolean fitsIntoType(IBasicType basicTarget, long n) {
 	private static long getApproximateSize(IBasicType type) {
 		switch (type.getKind()) {
 		case eChar:
+		case eChar8:
 			return 1;
 		case eWChar:
 			return 2;

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/SizeofCalculator.java

@@ -230,6 +230,7 @@ private SizeAndAlignment sizeAndAlignment(IBasicType type) {
 		case eBoolean:
 			return sizeof_bool;
 		case eChar:
+		case eChar8:
 			return SIZE_1;
 		case eInt:
 			return type.isShort() ? sizeof_short

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/CPPASTLiteralExpression.java

@@ -84,17 +84,23 @@ public class CPPASTLiteralExpression extends ASTNode implements ICPPASTLiteralEx
 	private ICPPEvaluation fEvaluation;
 	private IBinding fUserDefinedLiteralOperator;
 	private IASTImplicitName[] fImplicitNames;
+	private final boolean fUseChar8Type;
 
 	public CPPASTLiteralExpression(int kind, char[] value) {
-		this(kind, value, CharArrayUtils.EMPTY);
+		this(kind, value, CharArrayUtils.EMPTY, false);
 	}
 
-	public CPPASTLiteralExpression(int kind, char[] value, char[] numericCompilerSuffixes) {
+	public CPPASTLiteralExpression(int kind, char[] value, boolean useChar8Type) {
+		this(kind, value, CharArrayUtils.EMPTY, useChar8Type);
+	}
+
+	public CPPASTLiteralExpression(int kind, char[] value, char[] numericCompilerSuffixes, boolean useChar8Type) {
 		fKind = kind;
 		fSuffix = getSuffix(kind, value, CharArrayUtils.EMPTY);
 		fLiteral = getLiteral(value, fSuffix);
 		fNumericCompilerSuffixes = (numericCompilerSuffixes == null) ? CharArrayUtils.EMPTY : numericCompilerSuffixes;
 		fStringLiteralSize = -1;
+		fUseChar8Type = useChar8Type;
 	}
 
 	private CPPASTLiteralExpression(CPPASTLiteralExpression other) {
@@ -106,6 +112,7 @@ private CPPASTLiteralExpression(CPPASTLiteralExpression other) {
 		fEvaluation = other.fEvaluation;
 		fUserDefinedLiteralOperator = other.fUserDefinedLiteralOperator;
 		fImplicitNames = other.fImplicitNames;
+		fUseChar8Type = other.fUseChar8Type;
 	}
 
 	@Override
@@ -437,7 +444,13 @@ public Kind getBasicCharKind() {
 			return Kind.eChar32;
 		case 'u':
 			// Bug 526724 u8 should result in Kind.eChar
-			if (fLiteral[1] != '8') {
+			if (fLiteral[1] == '8') {
+				if (fUseChar8Type) {
+					return Kind.eChar8;
+				} else {
+					return Kind.eChar;
+				}
+			} else {
 				return Kind.eChar16;
 			}
 			//$FALL-THROUGH$

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/CPPASTSimpleDeclSpecifier.java

@@ -85,6 +85,8 @@ private int getType(Kind kind) {
 			return t_char;
 		case eWChar:
 			return t_wchar_t;
+		case eChar8:
+			return t_char8_t;
 		case eChar16:
 			return t_char16_t;
 		case eChar32:

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/CPPBasicType.java

@@ -127,6 +127,8 @@ static Kind getKind(final int simpleDeclSpecType) {
 			return Kind.eChar;
 		case IASTSimpleDeclSpecifier.t_wchar_t:
 			return Kind.eWChar;
+		case IASTSimpleDeclSpecifier.t_char8_t:
+			return Kind.eChar8;
 		case IASTSimpleDeclSpecifier.t_char16_t:
 			return Kind.eChar16;
 		case IASTSimpleDeclSpecifier.t_char32_t:
@@ -342,6 +344,7 @@ public int getType() {
 		case eBoolean:
 			return t_bool;
 		case eChar:
+		case eChar8:
 		case eChar16:
 		case eChar32:
 			return t_char;

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/CPPNodeFactory.java

@@ -584,12 +584,17 @@ public ICPPASTLinkageSpecification newLinkageSpecification(String literal) {
 
 	@Override
 	public ICPPASTLiteralExpression newLiteralExpression(int kind, String rep) {
-		return new CPPASTLiteralExpression(kind, rep.toCharArray());
+		return new CPPASTLiteralExpression(kind, rep.toCharArray(), false);
+	}
+
+	@Override
+	public ICPPASTLiteralExpression newLiteralExpression(int kind, String rep, boolean useChar8Type) {
+		return new CPPASTLiteralExpression(kind, rep.toCharArray(), useChar8Type);
 	}
 
 	@Override
 	public ICPPASTLiteralExpression newLiteralExpression(int kind, String rep, char[] numericCompilerSuffixes) {
-		return new CPPASTLiteralExpression(kind, rep.toCharArray(), numericCompilerSuffixes);
+		return new CPPASTLiteralExpression(kind, rep.toCharArray(), numericCompilerSuffixes, false);
 	}
 
 	@Override