diff --git a/expression/builtin_compare.go b/expression/builtin_compare.go
index 7ee32a040585c..bd5e7867c91ea 100644
--- a/expression/builtin_compare.go
+++ b/expression/builtin_compare.go
@@ -1348,8 +1348,13 @@ func (c *compareFunctionClass) refineArgs(ctx sessionctx.Context, args []Express
 	// int non-constant [cmp] non-int constant
 	if arg0IsInt && !arg0IsCon && !arg1IsInt && arg1IsCon {
 		arg1, isExceptional = RefineComparedConstant(ctx, *arg0Type, arg1, c.op)
-		finalArg1 = arg1
-		if isExceptional && arg1.GetType().EvalType() == types.ETInt {
+		// Why check not null flag
+		// eg: int_col > const_val(which is less than min_int32)
+		// If int_col got null, compare result cannot be true
+		if !isExceptional || (isExceptional && mysql.HasNotNullFlag(arg0Type.Flag)) {
+			finalArg1 = arg1
+		}
+		if isExceptional && arg1.GetType().EvalType() == types.ETInt && mysql.HasNotNullFlag(arg0Type.Flag) {
 			// Judge it is inf or -inf
 			// For int:
 			//			inf:  01111111 & 1 == 1
@@ -1367,8 +1372,10 @@ func (c *compareFunctionClass) refineArgs(ctx sessionctx.Context, args []Express
 	// non-int constant [cmp] int non-constant
 	if arg1IsInt && !arg1IsCon && !arg0IsInt && arg0IsCon {
 		arg0, isExceptional = RefineComparedConstant(ctx, *arg1Type, arg0, symmetricOp[c.op])
-		finalArg0 = arg0
-		if isExceptional && arg0.GetType().EvalType() == types.ETInt {
+		if !isExceptional || (isExceptional && mysql.HasNotNullFlag(arg1Type.Flag)) {
+			finalArg0 = arg0
+		}
+		if isExceptional && arg0.GetType().EvalType() == types.ETInt && mysql.HasNotNullFlag(arg1Type.Flag) {
 			if arg0.Value.GetInt64()&1 == 1 {
 				isNegativeInfinite = true
 			} else {
diff --git a/expression/builtin_compare_test.go b/expression/builtin_compare_test.go
index 4e8f16415f06b..9d11d8b5ad18a 100644
--- a/expression/builtin_compare_test.go
+++ b/expression/builtin_compare_test.go
@@ -27,7 +27,7 @@ import (
 )
 
 func (s *testEvaluatorSuite) TestCompareFunctionWithRefine(c *C) {
-	tblInfo := newTestTableBuilder("").add("a", mysql.TypeLong).build()
+	tblInfo := newTestTableBuilder("").add("a", mysql.TypeLong, mysql.NotNullFlag).build()
 	tests := []struct {
 		exprStr string
 		result  string
diff --git a/expression/expression_test.go b/expression/expression_test.go
index 2ecdc12f465eb..40eed2c946207 100644
--- a/expression/expression_test.go
+++ b/expression/expression_test.go
@@ -35,7 +35,7 @@ func (s *testEvaluatorSuite) TestNewValuesFunc(c *C) {
 }
 
 func (s *testEvaluatorSuite) TestEvaluateExprWithNull(c *C) {
-	tblInfo := newTestTableBuilder("").add("col0", mysql.TypeLonglong).add("col1", mysql.TypeLonglong).build()
+	tblInfo := newTestTableBuilder("").add("col0", mysql.TypeLonglong, 0).add("col1", mysql.TypeLonglong, 0).build()
 	schema := tableInfoToSchemaForTest(tblInfo)
 	col0 := schema.Columns[0]
 	col1 := schema.Columns[1]
@@ -142,15 +142,17 @@ type testTableBuilder struct {
 	tableName   string
 	columnNames []string
 	tps         []byte
+	flags       []uint
 }
 
 func newTestTableBuilder(tableName string) *testTableBuilder {
 	return &testTableBuilder{tableName: tableName}
 }
 
-func (builder *testTableBuilder) add(name string, tp byte) *testTableBuilder {
+func (builder *testTableBuilder) add(name string, tp byte, flag uint) *testTableBuilder {
 	builder.columnNames = append(builder.columnNames, name)
 	builder.tps = append(builder.tps, tp)
+	builder.flags = append(builder.flags, flag)
 	return builder
 }
 
@@ -165,6 +167,7 @@ func (builder *testTableBuilder) build() *model.TableInfo {
 		fieldType := types.NewFieldType(tp)
 		fieldType.Flen, fieldType.Decimal = mysql.GetDefaultFieldLengthAndDecimal(tp)
 		fieldType.Charset, fieldType.Collate = types.DefaultCharsetForType(tp)
+		fieldType.Flag = builder.flags[i]
 		ti.Columns = append(ti.Columns, &model.ColumnInfo{
 			ID:        int64(i + 1),
 			Name:      model.NewCIStr(colName),
diff --git a/expression/integration_test.go b/expression/integration_test.go
index 36cd761b732e7..5c461b8dff08c 100644
--- a/expression/integration_test.go
+++ b/expression/integration_test.go
@@ -8928,3 +8928,12 @@ func (s *testIntegrationSerialSuite) TestCollationForBinaryLiteral(c *C) {
 	tk.MustQuery("select * from t where col1 not in (0x1B,0x20) order by col1").Check(testkit.Rows("\x1e \xec 6966939640596047133"))
 	tk.MustExec("drop table t")
 }
+
+func (s *testIntegrationSuite) TestIssue23623(c *C) {
+	tk := testkit.NewTestKit(c, s.store)
+	tk.MustExec("use test")
+	tk.MustExec("drop table if exists t1;")
+	tk.MustExec("create table t1(c1 int);")
+	tk.MustExec("insert into t1 values(-2147483648), (-2147483648), (null);")
+	tk.MustQuery("select count(*) from t1 where c1 > (select sum(c1) from t1);").Check(testkit.Rows("2"))
+}
diff --git a/planner/core/cbo_test.go b/planner/core/cbo_test.go
index 0f085282fe103..2dcbe0ca2510d 100644
--- a/planner/core/cbo_test.go
+++ b/planner/core/cbo_test.go
@@ -453,7 +453,7 @@ func (s *testAnalyzeSuite) TestPreparedNullParam(c *C) {
 		testKit := testkit.NewTestKit(c, store)
 		testKit.MustExec("use test")
 		testKit.MustExec("drop table if exists t")
-		testKit.MustExec("create table t (id int, KEY id (id))")
+		testKit.MustExec("create table t (id int not null, KEY id (id))")
 		testKit.MustExec("insert into t values (1), (2), (3)")
 
 		sql := "select * from t where id = ?"