Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix copy some directory table to problems. #550

Merged
merged 2 commits into from
Aug 13, 2024
Merged

Fix copy some directory table to problems. #550

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
228dade
Fix copy some directory table to problems.
wenchaozhang-123 Aug 1, 2024
854a3a8
Fix psql of copy binary directory table to.
wenchaozhang-123 Aug 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions gpMgmt/bin/gpdirtableload
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ def parseargs():
help='Host to connect to')
parser.add_argument('--input-file', help='In upload mode, this means input files or '
'directory, while in download mode, means '
'which directory table to download')
'which directory table file to download')

parser.add_argument('--logfile', help='Log output to logfile')

Expand Down Expand Up @@ -406,7 +406,7 @@ class gpdirtableload:
try:
for file in self.allFiles:
cmdstr = cmdstrbase
cmdstr += '-c \"copy binary %s from \'%s\' ' % (self.options.table, file)
cmdstr += '-c \"\\copy binary %s from \'%s\' ' % (self.options.table, file)
if self.isDirectoryMode():
cmdstr += '\'%s/%s\' ' % (self.options.dest_path, os.path.relpath(file))
else:
Expand Down Expand Up @@ -485,7 +485,7 @@ class gpdirtableload:
os.makedirs(filedir, exist_ok=True)

cmdstr = cmdstrbase
cmdstr += '-c \"copy binary directory table %s \'%s\' to \'%s\' \"' % (self.options.table, file, fullpath)
cmdstr += '-c \"\\copy binary directory table %s \'%s\' to \'%s\' \"' % (self.options.table, file, fullpath)

cmd = Command(name='download directory table', ctxt=LOCAL, cmdStr=cmdstr)
self.pool.addCommand(cmd)
Expand Down
2 changes: 1 addition & 1 deletion src/backend/commands/copy.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,7 @@ DoCopy(ParseState *pstate, const CopyStmt *stmt,
errhint("Anyone can COPY to stdout or from stdin. "
"psql's \\copy command also works for anyone.")));

if (!is_from && !is_member_of_role(GetUserId(), ROLE_PG_WRITE_SERVER_FILES))
if (!is_from && !is_member_of_role(GetUserId(), ROLE_PG_WRITE_SERVER_FILES) && rel->rd_rel->relkind != RELKIND_DIRECTORY_TABLE)
ereport(ERROR,
(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
errmsg("must be superuser or a member of the pg_write_server_files role to COPY to a file"),
Expand Down
20 changes: 9 additions & 11 deletions src/bin/psql/copy.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,11 +152,19 @@ parse_slash_copy(const char *args)

xstrcat(&result->before_tofrom, " ");
xstrcat(&result->before_tofrom, token);
token = strtokx(NULL, whitespace, ".,()", "\"",
token = strtokx(NULL, whitespace, NULL, "\"",
0, false, false, pset.encoding);

if (!token)
goto error;

xstrcat(&result->before_tofrom, " ");
xstrcat(&result->before_tofrom, token);
token = strtokx(NULL, whitespace, NULL, "\"",
0, false, false, pset.encoding);

if (!token)
goto error;
}

xstrcat(&result->before_tofrom, " ");
Expand Down Expand Up @@ -207,16 +215,6 @@ parse_slash_copy(const char *args)
goto error;
}

if (token[0] == '\'')
{
xstrcat(&result->before_tofrom, " ");
xstrcat(&result->before_tofrom, token);
token = strtokx(NULL, whitespace, "()", "\"",
0, false, false, pset.encoding);
if (!token)
goto error;
}

if (pg_strcasecmp(token, "from") == 0)
result->from = true;
else if (pg_strcasecmp(token, "to") == 0)
Expand Down
36 changes: 23 additions & 13 deletions src/test/regress/input/directory_table.source
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,7 @@ CREATE DIRECTORY TABLE IF NOT EXISTS dir_table4 TABLESPACE directory_tblspc;
CREATE DIRECTORY TABLE IF NOT EXISTS dir_table2 TABLESPACE directory_tblspc; -- fail
CREATE DIRECTORY TABLE dir_table5 TABLESPACE directory_tblspc;
CREATE DIRECTORY TABLE dir_table6 TABLESPACE pg_default;
CREATE DIRECTORY TABLE "abs.dir_table";

SELECT count(*) FROM pg_directory_table;
SELECT relname, relisshared, relpersistence, relkind FROM pg_class WHERE relname like '%dir_table%' ORDER BY 1;
Expand Down Expand Up @@ -291,12 +292,6 @@ create trigger trigtest_a_stmt_tg_dirtable_1 after insert or update or delete on
for each statement execute procedure triggertest();

-- Test COPY DIRECTORY TABLE syntax
CREATE TEMP TABLE temp_dir(a int, b varchar) DISTRIBUTED BY (a);
INSERT INTO temp_dir select i, repeat('fwqfasdfqjeofqhewoufhqewfphpdsadjfhqewufhqw0efpqwefuhaeufpqepfiuhqefqewpfqehwoufqpwefhqwpuefhqwpi
ufdsahfpqefiqefqehfuqwhfoqhwoefuhqwouefhqwouefsakjfnqne34289340128rhqepefn10u1fu910fu14h01hhr104hrhewoqeur10qiwuehrp1j30rp1jrojpruhuqoeh
1-r-r4ajrgn52-g248uf24u3ifq', 9) from generate_series(1, 4999999) i;
COPY temp_dir TO '@abs_srcdir@/data/temp.csv';

SELECT relative_path, size, tag FROM dir_table1 ORDER BY 1;
SELECT relative_path, size, tag FROM dir_table2 ORDER BY 1;

Expand All @@ -312,8 +307,8 @@ SELECT relative_path, size, tag FROM dir_table2 ORDER BY 1;
\COPY BINARY dir_table1 FROM '@abs_srcdir@/data/nation.csv' 'nation3' WITH TAG 'nation2'; -- fail
\COPY BINARY dir_table1 FROM '@abs_srcdir@/data/nation.csv' 'nation4' WITH TAG 'nation';
\COPY BINARY dir_table1 FROM '@abs_srcdir@/data/nation.csv' 'nation5' WITH TAG 'nation' WITH TAG 'nation2'; -- fail
\COPY BINARY dir_table1 FROM '@abs_srcdir@/data/temp.csv' 'temp' WITH TAG 'TEMP'; -- OK
SELECT relative_path, size, tag FROM dir_table1 ORDER BY 1;
SELECT relative_path, content FROM directory_table('dir_table1') ORDER BY 1;

COPY dir_table2 FROM PROGRAM 'cat @abs_srcdir@/data/nation.csv'; -- fail
COPY dir_table2 FROM PROGRAM 'cat @abs_srcdir@/data/nation.csv' 'nation'; -- fail
Expand All @@ -325,8 +320,13 @@ COPY BINARY dir_table2 FROM PROGRAM 'cat @abs_srcdir@/data/nation.csv' 'nation3'
COPY BINARY dir_table2 FROM PROGRAM 'cat @abs_srcdir@/data/nation.csv' 'nation3' WITH TAG 'nation2'; -- fail
COPY BINARY dir_table2 FROM PROGRAM 'cat @abs_srcdir@/data/nation.csv' 'nation4' WITH TAG 'nation';
COPY BINARY dir_table2 FROM PROGRAM 'cat @abs_srcdir@/data/nation.csv' 'nation5' WITH TAG 'nation' WITH TAG 'nation2'; -- fail
COPY BINARY dir_table2 FROM '@abs_srcdir@/data/temp.csv' 'temp'; -- OK
SELECT relative_path, size, tag FROM dir_table2 ORDER BY 1;
SELECT relative_path, content FROM directory_table('dir_table2') ORDER BY 1;

\COPY BINARY dir_table1 FROM '@abs_srcdir@/data/nation.csv' 'nation.txt'; -- OK
COPY BINARY dir_table1 FROM '@abs_srcdir@/data/nation.csv' 'nation2.txt'; -- OK
\COPY BINARY "abs.dir_table" FROM '@abs_srcdir@/data/nation.csv' 'aa.bb'; -- OK
COPY BINARY "abs.dir_table" FROM '@abs_srcdir@/data/nation.csv' 'cc.dd'; -- OK

-- Test copy binary from directory table
\COPY BINARY dir_table1 FROM '@abs_srcdir@/data/nation.csv' 'nation_failed' (format CSV);
Expand Down Expand Up @@ -361,9 +361,6 @@ COPY BINARY dir_table2 FROM '@abs_srcdir@/data/nation.csv' 'nation_failed' (forc
COPY BINARY dir_table2 FROM '@abs_srcdir@/data/nation.csv' 'nation_failed' (convert_selectively (a));
COPY BINARY dir_table2 FROM '@abs_srcdir@/data/nation.csv' 'nation_failed' (encoding 'sql_ascii');

SELECT remove_file('dir_table1', 'temp');
SELECT remove_file('dir_table2', 'temp');

-- Test copy file content md5
CREATE OR REPLACE FUNCTION file_content(text, text) RETURNS BYTEA LANGUAGE SQL AS
'select content from directory_table($1) where relative_path = $2';
Expand Down Expand Up @@ -406,28 +403,41 @@ COPY BINARY DIRECTORY TABLE dir_table1 'nation2' TO stdout; -- OK
\COPY BINARY DIRECTORY TABLE dir_table1 'nation2' TO PROGRAM 'gzip -c -1 > @abs_srcdir@/data/nation2.gz'; -- OK
COPY BINARY DIRECTORY TABLE dir_table1 'nation2' TO PROGRAM 'gzip -c -1 > @abs_srcdir@/data/nation2.gz'; -- OK

\COPY BINARY DIRECTORY TABLE "abs.dir_table" 'aa.bb' TO '@abs_srcdir@/data/aa.bb'; -- OK
COPY BINARY DIRECTORY TABLE "abs.dir_table" 'cc.dd' TO '@abs_srcdir@/data/cc.dd'; -- OK
\COPY BINARY DIRECTORY TABLE dir_table1 'nation.txt' TO '@abs_srcdir@/data/nation.txt'; -- OK
COPY BINARY DIRECTORY TABLE dir_table1 'nation2.txt' TO '@abs_srcdir@/data/nation2.txt'; -- OK
\COPY BINARY DIRECTORY TABLE public.dir_table1 'nation.txt' TO '@abs_srcdir@/data/nation3.txt'; -- OK
COPY BINARY DIRECTORY TABLE public.dir_table1 'nation2.txt' TO '@abs_srcdir@/data/nation4.txt'; -- OK


SELECT relative_path, size, tag FROM dir_table1 ORDER BY 1;
SELECT relative_path, size, tag FROM dir_table2 ORDER BY 1;

-- Test join between two directory schema tables
ANALYZE dir_table1;
ANALYZE dir_table2;

EXPLAIN (COSTS OFF) SELECT dir_table1.relative_path FROM dir_table1, dir_table2
WHERE dir_table1.relative_path = dir_table2.relative_path ORDER BY 1;
SELECT dir_table1.relative_path FROM dir_table1, dir_table2
WHERE dir_table1.relative_path = dir_table2.relative_path ORDER BY 1;

ANALYZE dir_table1;
ANALYZE dir_table2;
EXPLAIN (COSTS OFF) SELECT dir_table1.relative_path FROM dir_table1, dir_table2
WHERE dir_table1.size = dir_table2.size ORDER BY 1 LIMIT 1;
SELECT dir_table1.relative_path FROM dir_table1, dir_table2
WHERE dir_table1.size = dir_table2.size ORDER BY 1 LIMIT 1;

ANALYZE dir_table1;
ANALYZE dir_table2;
EXPLAIN (COSTS OFF) SELECT dir_table1.relative_path FROM dir_table1, dir_table2
WHERE dir_table1.md5 = dir_table2.md5 ORDER BY 1 LIMIT 1;
SELECT dir_table1.relative_path FROM dir_table1, dir_table2
WHERE dir_table1.md5 = dir_table2.md5 ORDER BY 1 LIMIT 1;

ANALYZE dir_table1;
ANALYZE dir_table2;
EXPLAIN (COSTS OFF) SELECT dir_table1.relative_path FROM dir_table1, dir_table2
WHERE dir_table1.tag = dir_table2.tag ORDER BY 1;
SELECT dir_table1.relative_path FROM dir_table1, dir_table2
Expand Down Expand Up @@ -621,6 +631,7 @@ DROP DIRECTORY TABLE IF EXISTS dir_table3;
DROP DIRECTORY TABLE IF EXISTS dir_table4;
DROP DIRECTORY TABLE IF EXISTS dir_table5;
DROP DIRECTORY TABLE IF EXISTS dir_table6;
DROP DIRECTORY TABLE IF EXISTS "abs.dir_table";

DROP FUNCTION IF EXISTS triggertest;

Expand Down Expand Up @@ -667,6 +678,5 @@ DROP TRIGGER IF EXISTS trigtest_b_stmt_tg_dirtable_1 ON dir_table1;
DROP TRIGGER IF EXISTS trigtest_a_stmt_tg_dirtable_1 ON dir_table1;

\!rm -rf @testtablespace@;
\!rm -rf @abs_srcdir@/data/temp.csv

DROP TABLESPACE directory_tblspc;
Loading
Loading