Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[SPARK-39335][SQL] DescribeTableCommand should redact properties #36722

Closed
wants to merge 3 commits into from
Closed

[SPARK-39335][SQL] DescribeTableCommand should redact properties #36722

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
65401dd
[SPARK-39335][SQL] DescribeTableCommand should redact properties.
AngersZhuuuu May 30, 2022
88b0cf0
Update describe.sql.out
AngersZhuuuu May 30, 2022
8d90c2b
Update interface.scala
AngersZhuuuu May 30, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 6 additions & 5 deletions sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,8 @@ case class CatalogTablePartition(
map.put("Partition Values", s"[$specString]")
map ++= storage.toLinkedHashMap
if (parameters.nonEmpty) {
map.put("Partition Parameters", s"{${parameters.map(p => p._1 + "=" + p._2).mkString(", ")}}")
map.put("Partition Parameters", s"{" +
s"${SQLConf.get.redactOptions(parameters).map(p => p._1 + "=" + p._2).mkString(", ")}}")
}
map.put("Created Time", new Date(createTime).toString)
val lastAccess = {
Expand Down Expand Up @@ -383,10 +384,10 @@ case class CatalogTable(

def toLinkedHashMap: mutable.LinkedHashMap[String, String] = {
val map = new mutable.LinkedHashMap[String, String]()
val tableProperties = properties
.filterKeys(!_.startsWith(VIEW_PREFIX))
.toSeq.sortBy(_._1)
.map(p => p._1 + "=" + p._2)
val tableProperties =
SQLConf.get.redactOptions(properties.filterKeys(!_.startsWith(VIEW_PREFIX)).toMap)
.toSeq.sortBy(_._1)
.map(p => p._1 + "=" + p._2)
val partitionColumns = partitionColumnNames.map(quoteIdentifier).mkString("[", ", ", "]")
val lastAccess = {
if (lastAccessTime <= 0) "UNKNOWN" else new Date(lastAccessTime).toString
Expand Down
4 changes: 2 additions & 2 deletions sql/core/src/test/resources/sql-tests/inputs/describe.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
CREATE TABLE t (a STRING, b INT, c STRING, d STRING) USING parquet
OPTIONS (a '1', b '2')
OPTIONS (a '1', b '2', password 'password')
PARTITIONED BY (c, d) CLUSTERED BY (a) SORTED BY (b ASC) INTO 2 BUCKETS
COMMENT 'table_comment'
TBLPROPERTIES (t 'test');
TBLPROPERTIES (t 'test', password 'password');

CREATE TEMPORARY VIEW temp_v AS SELECT * FROM t;

Expand Down
32 changes: 16 additions & 16 deletions sql/core/src/test/resources/sql-tests/results/describe.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,10 @@

-- !query
CREATE TABLE t (a STRING, b INT, c STRING, d STRING) USING parquet
OPTIONS (a '1', b '2')
OPTIONS (a '1', b '2', password 'password')
PARTITIONED BY (c, d) CLUSTERED BY (a) SORTED BY (b ASC) INTO 2 BUCKETS
COMMENT 'table_comment'
TBLPROPERTIES (t 'test')
TBLPROPERTIES (t 'test', password 'password')
-- !query schema
struct<>
-- !query output
Expand Down Expand Up @@ -130,9 +130,9 @@ Num Buckets 2
Bucket Columns [`a`]
Sort Columns [`b`]
Comment table_comment
Table Properties [e=3, t=test]
Table Properties [e=3, password=*********(redacted), t=test]
Location [not included in comparison]/{warehouse_dir}/t
Storage Properties [a=1, b=2]
Storage Properties [a=1, b=2, password=*********(redacted)]
Partition Provider Catalog


Expand Down Expand Up @@ -162,9 +162,9 @@ Num Buckets 2
Bucket Columns [`a`]
Sort Columns [`b`]
Comment table_comment
Table Properties [e=3, t=test]
Table Properties [e=3, password=*********(redacted), t=test]
Location [not included in comparison]/{warehouse_dir}/t
Storage Properties [a=1, b=2]
Storage Properties [a=1, b=2, password=*********(redacted)]
Partition Provider Catalog


Expand Down Expand Up @@ -202,9 +202,9 @@ Num Buckets 2
Bucket Columns [`a`]
Sort Columns [`b`]
Comment table_comment
Table Properties [t=test]
Table Properties [password=*********(redacted), t=test]
Location [not included in comparison]/{warehouse_dir}/t
Storage Properties [a=1, b=2]
Storage Properties [a=1, b=2, password=*********(redacted)]
Partition Provider Catalog


Expand Down Expand Up @@ -241,9 +241,9 @@ Provider parquet
Num Buckets 2
Bucket Columns [`a`]
Sort Columns [`b`]
Table Properties [t=test]
Table Properties [password=*********(redacted), t=test]
Location [not included in comparison]/{warehouse_dir}/t
Storage Properties [a=1, b=2]
Storage Properties [a=1, b=2, password=*********(redacted)]
Partition Provider Catalog


Expand Down Expand Up @@ -281,7 +281,7 @@ Database default
Table t
Partition Values [c=Us, d=1]
Location [not included in comparison]/{warehouse_dir}/t/c=Us/d=1
Storage Properties [a=1, b=2]
Storage Properties [a=1, b=2, password=*********(redacted)]
Created Time [not included in comparison]
Last Access [not included in comparison]

Expand All @@ -290,7 +290,7 @@ Num Buckets 2
Bucket Columns [`a`]
Sort Columns [`b`]
Location [not included in comparison]/{warehouse_dir}/t
Storage Properties [a=1, b=2]
Storage Properties [a=1, b=2, password=*********(redacted)]


-- !query
Expand All @@ -312,7 +312,7 @@ Database default
Table t
Partition Values [c=Us, d=1]
Location [not included in comparison]/{warehouse_dir}/t/c=Us/d=1
Storage Properties [a=1, b=2]
Storage Properties [a=1, b=2, password=*********(redacted)]
Created Time [not included in comparison]
Last Access [not included in comparison]

Expand All @@ -321,7 +321,7 @@ Num Buckets 2
Bucket Columns [`a`]
Sort Columns [`b`]
Location [not included in comparison]/{warehouse_dir}/t
Storage Properties [a=1, b=2]
Storage Properties [a=1, b=2, password=*********(redacted)]


-- !query
Expand All @@ -343,7 +343,7 @@ Database default
Table t
Partition Values [c=Us, d=1]
Location [not included in comparison]/{warehouse_dir}/t/c=Us/d=1
Storage Properties [a=1, b=2]
Storage Properties [a=1, b=2, password=*********(redacted)]
Created Time [not included in comparison]
Last Access [not included in comparison]

Expand All @@ -352,7 +352,7 @@ Num Buckets 2
Bucket Columns [`a`]
Sort Columns [`b`]
Location [not included in comparison]/{warehouse_dir}/t
Storage Properties [a=1, b=2]
Storage Properties [a=1, b=2, password=*********(redacted)]


-- !query
Expand Down