Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix/deprecation extractall #16918

Merged
merged 8 commits into from
Sep 26, 2024
1 change: 1 addition & 0 deletions conan/api/subapi/cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,7 @@ def restore(self, path):
the_tar = tarfile.open(fileobj=file_handler)
fileobj = the_tar.extractfile("pkglist.json")
pkglist = fileobj.read()
the_tar.extraction_filter = (lambda member, _: member) # fully_trusted (Py 3.14)
the_tar.extractall(path=cache_folder)
the_tar.close()

Expand Down
19 changes: 13 additions & 6 deletions conan/tools/files/files.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ def rm(conanfile, pattern, folder, recursive=False, excludes=None):

def get(conanfile, url, md5=None, sha1=None, sha256=None, destination=".", filename="",
keep_permissions=False, pattern=None, verify=True, retry=None, retry_wait=None,
auth=None, headers=None, strip_root=False):
auth=None, headers=None, strip_root=False, extract_filter=None):
"""
High level download and decompressing of a tgz, zip or other compressed format file.
Just a high level wrapper for download, unzip, and remove the temporary zip file once unzipped.
Expand All @@ -115,6 +115,7 @@ def get(conanfile, url, md5=None, sha1=None, sha256=None, destination=".", filen
:param auth: forwarded to ``tools.file.download()``.
:param headers: forwarded to ``tools.file.download()``.
:param strip_root: forwarded to ``tools.file.unzip()``.
:param extract_filter: forwarded to ``tools.file.unzip()``.
"""

if not filename: # deduce filename from the URL
Expand All @@ -128,7 +129,7 @@ def get(conanfile, url, md5=None, sha1=None, sha256=None, destination=".", filen
retry=retry, retry_wait=retry_wait, auth=auth, headers=headers,
md5=md5, sha1=sha1, sha256=sha256)
unzip(conanfile, filename, destination=destination, keep_permissions=keep_permissions,
pattern=pattern, strip_root=strip_root)
pattern=pattern, strip_root=strip_root, extract_filter=extract_filter)
os.unlink(filename)


Expand Down Expand Up @@ -262,7 +263,7 @@ def chdir(conanfile, newdir):


def unzip(conanfile, filename, destination=".", keep_permissions=False, pattern=None,
strip_root=False):
strip_root=False, extract_filter=None):
"""
Extract different compressed formats

Expand All @@ -277,14 +278,18 @@ def unzip(conanfile, filename, destination=".", keep_permissions=False, pattern=
This should be a Unix shell-style wildcard, see fnmatch documentation for more details.
:param strip_root: (Optional, Defaulted to False) If True, and all the unzipped contents are
in a single folder it will flat the folder moving all the contents to the parent folder.
:param extract_filter: (Optional, defaulted to None). When extracting a tar file,
use the tar extracting filters define by Python in
https://docs.python.org/3/library/tarfile.html
"""

output = conanfile.output
extract_filter = conanfile.conf.get("tools.files.unzip:filter") or extract_filter
output.info(f"Unzipping {filename} to {destination}")
if (filename.endswith(".tar.gz") or filename.endswith(".tgz") or
filename.endswith(".tbz2") or filename.endswith(".tar.bz2") or
filename.endswith(".tar")):
return untargz(filename, destination, pattern, strip_root)
return untargz(filename, destination, pattern, strip_root, extract_filter)
if filename.endswith(".gz"):
target_name = filename[:-3] if destination == "." else destination
target_dir = os.path.dirname(target_name)
Expand All @@ -295,7 +300,7 @@ def unzip(conanfile, filename, destination=".", keep_permissions=False, pattern=
shutil.copyfileobj(fin, fout)
return
if filename.endswith(".tar.xz") or filename.endswith(".txz"):
return untargz(filename, destination, pattern, strip_root)
return untargz(filename, destination, pattern, strip_root, extract_filter)

import zipfile
full_path = os.path.normpath(os.path.join(os.getcwd(), destination))
Expand Down Expand Up @@ -363,10 +368,12 @@ def print_progress(_, __):
output.writeln("")


def untargz(filename, destination=".", pattern=None, strip_root=False):
def untargz(filename, destination=".", pattern=None, strip_root=False, extract_filter=None):
# NOT EXPOSED at `conan.tools.files` but used in tests
import tarfile
with tarfile.TarFile.open(filename, 'r:*') as tarredgzippedFile:
f = getattr(tarfile, f"{extract_filter}_filter", None) if extract_filter else None
tarredgzippedFile.extraction_filter = f or (lambda member_, _: member_)
if not pattern and not strip_root:
tarredgzippedFile.extractall(destination)
else:
Expand Down
1 change: 1 addition & 0 deletions conans/model/conf.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@
"tools.files.download:retry": "Number of retries in case of failure when downloading",
"tools.files.download:retry_wait": "Seconds to wait between download attempts",
"tools.files.download:verify": "If set, overrides recipes on whether to perform SSL verification for their downloaded files. Only recommended to be set while testing",
"tools.files.unzip:filter": "Define tar extraction filter: 'fully_trusted', 'tar', 'data'",
"tools.graph:vendor": "(Experimental) If 'build', enables the computation of dependencies of vendoring packages to build them",
"tools.graph:skip_binaries": "Allow the graph to skip binaries not needed in the current configuration (True by default)",
"tools.gnu:make_program": "Indicate path to make program",
Expand Down
1 change: 1 addition & 0 deletions conans/util/files.py
Original file line number Diff line number Diff line change
Expand Up @@ -296,6 +296,7 @@ def tar_extract(fileobj, destination_dir):
# NOTE: The errorlevel=2 has been removed because it was failing in Win10, it didn't allow to
# "could not change modification time", with time=0
# the_tar.errorlevel = 2 # raise exception if any error
the_tar.extraction_filter = (lambda member, path: member) # fully_trusted, avoid Py3.14 break
the_tar.extractall(path=destination_dir)
the_tar.close()

Expand Down
43 changes: 43 additions & 0 deletions test/integration/test_source_download_password.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,17 @@
import json
import os
import platform
import sys
import textwrap
from shutil import copy
from unittest import mock

import pytest

from conan.internal.api.uploader import compress_files
from conan.test.assets.genconanfile import GenConanfile
from conan.test.utils.file_server import TestFileServer
from conan.test.utils.test_files import temp_folder
from conan.test.utils.tools import TestClient
from conans.util.files import save

Expand Down Expand Up @@ -77,3 +84,39 @@ def test_source_credentials_only_download():
c.run("upload * -c -r=default")
c.run("remove * -c")
c.run("download pkg/0.1 -r=default")


@pytest.mark.skipif(sys.version_info.minor < 12 or platform.system() == "Windows",
reason="Extraction filters only Python 3.12, using symlinks (not Windows)")
def test_blocked_malicius_tgz():
folder = temp_folder()
f = os.path.join(folder, "myfile.txt")
save(f, "The contents")
s = os.path.join(folder, "mylink.txt")
os.symlink(f, s)
tgz_path = compress_files({f: f, s: s}, "myfiles.tgz", dest_dir=folder)
os.remove(f)

conan_file = textwrap.dedent("""
from conan import ConanFile
from conan.tools.files import get
class Pkg(ConanFile):
name = "pkg"
version = "0.1"
def source(self):
get(self, "http://fake_url/myfiles.tgz")
""")
client = TestClient()
client.save({"conanfile.py": conan_file})

with mock.patch("conan.tools.files.files.download") as mock_download:
def download_zip(*args, **kwargs): # noqa
copy(tgz_path, os.getcwd())
mock_download.side_effect = download_zip
client.run("create . -c tools.files.unzip:filter=data", assert_error=True)
assert "AbsoluteLinkError" in client.out
client.save({"conanfile.py": conan_file.format("extract_filter='fully_trusted'")})
client.run("create . ") # Doesn't fail now
# user conf has precedence
client.save({"conanfile.py": conan_file.format("extract_filter='data'")})
client.run("create . -c tools.files.unzip:filter=fully_trusted") # Doesn't fail now