Skip to content

Commit

Permalink
[SPARK-39335][SQL] DescribeTableCommand should redact properties
Browse files Browse the repository at this point in the history 
### What changes were proposed in this pull request?
In current code we only redact storage's properties, for normal properties we should redact too.

### Why are the changes needed?
Redact properties

### Does this PR introduce _any_ user-facing change?
After this pr, when describe table, table properties will be redacted.

### How was this patch tested?
Added UT

Closes #36722 from AngersZhuuuu/SPARK-39335.

Authored-by: Angerszhuuuu <angers.zhu@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
  • Loading branch information
@AngersZhuuuu @cloud-fan
AngersZhuuuu authored and cloud-fan committed May 31, 2022
1 parent 33bfef0 commit 8bbbdb5
Show file tree
Hide file tree
Showing 3 changed files with 24 additions and 23 deletions.
11 changes: 6 additions & 5 deletions sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,8 @@ case class CatalogTablePartition(
map.put("Partition Values", s"[$specString]")
map ++= storage.toLinkedHashMap
if (parameters.nonEmpty) {
map.put("Partition Parameters", s"{${parameters.map(p => p._1 + "=" + p._2).mkString(", ")}}")
map.put("Partition Parameters", s"{" +
s"${SQLConf.get.redactOptions(parameters).map(p => p._1 + "=" + p._2).mkString(", ")}}")
}
map.put("Created Time", new Date(createTime).toString)
val lastAccess = {
Expand Down Expand Up @@ -383,10 +384,10 @@ case class CatalogTable(

def toLinkedHashMap: mutable.LinkedHashMap[String, String] = {
val map = new mutable.LinkedHashMap[String, String]()
val tableProperties = properties
.filterKeys(!_.startsWith(VIEW_PREFIX))
.toSeq.sortBy(_._1)
.map(p => p._1 + "=" + p._2)
val tableProperties =
SQLConf.get.redactOptions(properties.filterKeys(!_.startsWith(VIEW_PREFIX)).toMap)
.toSeq.sortBy(_._1)
.map(p => p._1 + "=" + p._2)
val partitionColumns = partitionColumnNames.map(quoteIdentifier).mkString("[", ", ", "]")
val lastAccess = {
if (lastAccessTime <= 0) "UNKNOWN" else new Date(lastAccessTime).toString
Expand Down
4 changes: 2 additions & 2 deletions sql/core/src/test/resources/sql-tests/inputs/describe.sql
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
CREATE TABLE t (a STRING, b INT, c STRING, d STRING) USING parquet
OPTIONS (a '1', b '2')
OPTIONS (a '1', b '2', password 'password')
PARTITIONED BY (c, d) CLUSTERED BY (a) SORTED BY (b ASC) INTO 2 BUCKETS
COMMENT 'table_comment'
TBLPROPERTIES (t 'test');
TBLPROPERTIES (t 'test', password 'password');

CREATE TEMPORARY VIEW temp_v AS SELECT * FROM t;

Expand Down
32 changes: 16 additions & 16 deletions sql/core/src/test/resources/sql-tests/results/describe.sql.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,10 @@

-- !query
CREATE TABLE t (a STRING, b INT, c STRING, d STRING) USING parquet
OPTIONS (a '1', b '2')
OPTIONS (a '1', b '2', password 'password')
PARTITIONED BY (c, d) CLUSTERED BY (a) SORTED BY (b ASC) INTO 2 BUCKETS
COMMENT 'table_comment'
TBLPROPERTIES (t 'test')
TBLPROPERTIES (t 'test', password 'password')
-- !query schema
struct<>
-- !query output
Expand Down Expand Up @@ -130,9 +130,9 @@ Num Buckets 2
Bucket Columns [`a`]
Sort Columns [`b`]
Comment table_comment
Table Properties [e=3, t=test]
Table Properties [e=3, password=*********(redacted), t=test]
Location [not included in comparison]/{warehouse_dir}/t
Storage Properties [a=1, b=2]
Storage Properties [a=1, b=2, password=*********(redacted)]
Partition Provider Catalog


Expand Down Expand Up @@ -162,9 +162,9 @@ Num Buckets 2
Bucket Columns [`a`]
Sort Columns [`b`]
Comment table_comment
Table Properties [e=3, t=test]
Table Properties [e=3, password=*********(redacted), t=test]
Location [not included in comparison]/{warehouse_dir}/t
Storage Properties [a=1, b=2]
Storage Properties [a=1, b=2, password=*********(redacted)]
Partition Provider Catalog


Expand Down Expand Up @@ -202,9 +202,9 @@ Num Buckets 2
Bucket Columns [`a`]
Sort Columns [`b`]
Comment table_comment
Table Properties [t=test]
Table Properties [password=*********(redacted), t=test]
Location [not included in comparison]/{warehouse_dir}/t
Storage Properties [a=1, b=2]
Storage Properties [a=1, b=2, password=*********(redacted)]
Partition Provider Catalog


Expand Down Expand Up @@ -241,9 +241,9 @@ Provider parquet
Num Buckets 2
Bucket Columns [`a`]
Sort Columns [`b`]
Table Properties [t=test]
Table Properties [password=*********(redacted), t=test]
Location [not included in comparison]/{warehouse_dir}/t
Storage Properties [a=1, b=2]
Storage Properties [a=1, b=2, password=*********(redacted)]
Partition Provider Catalog


Expand Down Expand Up @@ -281,7 +281,7 @@ Database default
Table t
Partition Values [c=Us, d=1]
Location [not included in comparison]/{warehouse_dir}/t/c=Us/d=1
Storage Properties [a=1, b=2]
Storage Properties [a=1, b=2, password=*********(redacted)]
Created Time [not included in comparison]
Last Access [not included in comparison]

Expand All @@ -290,7 +290,7 @@ Num Buckets 2
Bucket Columns [`a`]
Sort Columns [`b`]
Location [not included in comparison]/{warehouse_dir}/t
Storage Properties [a=1, b=2]
Storage Properties [a=1, b=2, password=*********(redacted)]


-- !query
Expand All @@ -312,7 +312,7 @@ Database default
Table t
Partition Values [c=Us, d=1]
Location [not included in comparison]/{warehouse_dir}/t/c=Us/d=1
Storage Properties [a=1, b=2]
Storage Properties [a=1, b=2, password=*********(redacted)]
Created Time [not included in comparison]
Last Access [not included in comparison]

Expand All @@ -321,7 +321,7 @@ Num Buckets 2
Bucket Columns [`a`]
Sort Columns [`b`]
Location [not included in comparison]/{warehouse_dir}/t
Storage Properties [a=1, b=2]
Storage Properties [a=1, b=2, password=*********(redacted)]


-- !query
Expand All @@ -343,7 +343,7 @@ Database default
Table t
Partition Values [c=Us, d=1]
Location [not included in comparison]/{warehouse_dir}/t/c=Us/d=1
Storage Properties [a=1, b=2]
Storage Properties [a=1, b=2, password=*********(redacted)]
Created Time [not included in comparison]
Last Access [not included in comparison]

Expand All @@ -352,7 +352,7 @@ Num Buckets 2
Bucket Columns [`a`]
Sort Columns [`b`]
Location [not included in comparison]/{warehouse_dir}/t
Storage Properties [a=1, b=2]
Storage Properties [a=1, b=2, password=*********(redacted)]


-- !query
Expand Down

0 comments on commit 8bbbdb5

Please sign in to comment.