File: C:/Users/fred/anaconda3/Lib/site-packages/conda_pack/core.py
import glob
import json
import os
import re
import shlex
import shutil
import subprocess
import sys
import tempfile
import warnings
from contextlib import contextmanager
from datetime import datetime
from fnmatch import fnmatch
import pkg_resources
from ._progress import progressbar
from .compat import default_encoding, find_py_source, is_32bit, on_win
from .prefixes import SHEBANG_REGEX, replace_prefix
__all__ = ('CondaPackException', 'CondaEnv', 'File', 'pack')
class CondaPackException(Exception):
"""Internal exception to report to user"""
pass
# String is split so as not to appear in the file bytes unintentionally
PREFIX_PLACEHOLDER = ('/opt/anaconda1anaconda2'
'anaconda3')
BIN_DIR = 'Scripts' if on_win else 'bin'
_current_dir = os.path.dirname(__file__)
if on_win:
_scripts = [(os.path.join(_current_dir, 'scripts', 'windows', 'activate.bat'),
os.path.join(BIN_DIR, 'activate.bat')),
(os.path.join(_current_dir, 'scripts', 'windows', 'deactivate.bat'),
os.path.join(BIN_DIR, 'deactivate.bat'))]
else:
_scripts = [(os.path.join(_current_dir, 'scripts', 'posix', 'activate'),
os.path.join(BIN_DIR, 'activate')),
(os.path.join(_current_dir, 'scripts', 'posix', 'deactivate'),
os.path.join(BIN_DIR, 'deactivate'))]
class _Context:
def __init__(self):
self.is_cli = False
def warn(self, msg):
if self.is_cli:
print(msg + "\n", file=sys.stderr)
else:
warnings.warn(msg)
@contextmanager
def set_cli(self):
old = self.is_cli
try:
self.is_cli = True
yield
finally:
self.is_cli = old
context = _Context()
class CondaEnv:
"""A Conda environment for packaging.
Use :func:`CondaEnv.from_prefix`, :func:`CondaEnv.from_name`, or
:func:`CondaEnv.from_default` instead of the default constructor.
Attributes
----------
prefix : str
The path to the conda environment.
files : list of File
A list of :class:`File` objects representing all files in conda
environment.
Examples
--------
Package the environment ``foo`` into a zip archive:
>>> (CondaEnv.from_name("foo")
... .pack(output="foo.zip"))
"/full/path/to/foo.zip"
Package the environment ``foo`` into a parcel:
>>> (CondaEnv.from_prefix("/path/to/envs/foo")
... .pack(format="parcel", parcel_version="2020.09.01"))
"/full/path/to/foo-2020.09.01.parcel"
Package the current environment into a ``tar.gz`` archive:
>>> (CondaEnv.from_default()
... .pack(output="output.tar.gz"))
"/full/path/to/output.tar.gz"
Create a CondaEnv object from the current environment, excluding all
``*.pyx`` files, except those from ``cytoolz``.
>>> env = (CondaEnv.from_default()
... .exclude("*.pyx")
... .include("lib/python3.7/site-packages/cytoolz/*.pyx"))
CondaEnv<'~/miniconda/envs/example', 1234 files>
"""
def __init__(self, prefix, files, excluded_files=None):
self.prefix = prefix
self.files = files
self._excluded_files = excluded_files or []
def __repr__(self):
return 'CondaEnv<%r, %d files>' % (self.prefix, len(self))
def __len__(self):
return len(self.files)
def __iter__(self):
return iter(self.files)
@property
def name(self):
"""The name of the environment"""
return os.path.basename(self.prefix)
@classmethod
def from_name(cls, name, **kwargs):
"""Create a ``CondaEnv`` from a named environment.
Parameters
----------
name : str
The name of the conda environment.
Returns
-------
env : CondaEnv
"""
return cls.from_prefix(name_to_prefix(name), **kwargs)
@classmethod
def from_prefix(cls, prefix, **kwargs):
"""Create a ``CondaEnv`` from a given prefix.
Parameters
----------
prefix : str
The path to the conda environment.
Returns
-------
env : CondaEnv
"""
prefix = os.path.abspath(prefix)
files = load_environment(prefix, **kwargs)
return cls(prefix, files)
@classmethod
def from_default(cls, **kwargs):
"""Create a ``CondaEnv`` from the current environment.
Returns
-------
env : CondaEnv
"""
return cls.from_prefix(name_to_prefix(), **kwargs)
def exclude(self, pattern):
"""Exclude all files that match ``pattern`` from being packaged.
This can be useful to remove functionality that isn't needed in the
archive but is part of the original conda package.
Parameters
----------
pattern : str
A file pattern. May include shell-style wildcards a-la ``glob``.
Returns
-------
env : CondaEnv
A new env with any matching files excluded.
"""
files = []
excluded = list(self._excluded_files) # copy
include = files.append
exclude = excluded.append
for f in self.files:
if fnmatch(f.target, pattern):
exclude(f)
else:
include(f)
return CondaEnv(self.prefix, files, excluded)
def include(self, pattern):
"""Re-add all excluded files that match ``pattern``
Parameters
----------
pattern : str
A file pattern. May include shell-style wildcards a-la ``glob``.
Returns
-------
env : CondaEnv
A new env with any matching files that were previously excluded
re-included.
"""
files = list(self.files) # copy
excluded = []
include = files.append
exclude = excluded.append
for f in self._excluded_files:
if fnmatch(f.target, pattern):
include(f)
else:
exclude(f)
return CondaEnv(self.prefix, files, excluded)
def _output_and_format(self, output=None, format='infer'):
if output is None and format == 'infer':
format = 'tar.gz'
elif format == 'infer':
if output.endswith('.parcel'):
format = 'parcel'
elif output.endswith('.zip'):
format = 'zip'
elif output.endswith('.tar.gz') or output.endswith('.tgz'):
format = 'tar.gz'
elif output.endswith('.tar.bz2') or output.endswith('.tbz2'):
format = 'tar.bz2'
elif output.endswith('.tar.xz') or output.endswith('.txz'):
format = 'tar.xz'
elif output.endswith('.tar'):
format = 'tar'
elif output.endswith('.squashfs'):
format = 'squashfs'
else:
raise CondaPackException("Unknown file extension %r" % output)
elif format not in {'zip', 'tar.gz', 'tgz', 'tar.bz2', 'tbz2',
'tar.xz', 'txz', 'tar', 'parcel', 'squashfs'}:
raise CondaPackException("Unknown format %r" % format)
elif output is not None and output.endswith('.parcel'):
if format not in ('tar.gz', 'tgz'):
raise CondaPackException("Invalid format for parcel %r" % format)
format = 'parcel'
# Construct the parcel name outside of this function
if output is None and format != 'parcel':
output = os.extsep.join([self.name, format])
return output, format
def _parcel_output(self, parcel_root, parcel_name, parcel_version, parcel_distro):
parcel_root = parcel_root or '/opt/cloudera/parcels'
parcel_name = parcel_name or self.name
parcel_version = parcel_version or datetime.today().strftime(format='%Y.%m.%d')
parcel_distro = parcel_distro or 'el7'
if '-' in parcel_name:
raise CondaPackException("Parcel names may not have dashes: %s" % parcel_name)
if '-' in parcel_distro:
raise CondaPackException("Parcel distributions may not have dashes: %s" % parcel_distro)
arcroot = parcel_name + '-' + parcel_version
triple = arcroot + '-' + parcel_distro
dest_prefix = os.path.join(parcel_root, arcroot)
return dest_prefix, arcroot, triple
def pack(self, output=None, format='infer',
arcroot='', dest_prefix=None,
parcel_root=None, parcel_name=None,
parcel_version=None, parcel_distro=None,
verbose=False, force=False,
compress_level=4, n_threads=1,
zip_symlinks=False, zip_64=True):
"""Package the conda environment into an archive file.
Parameters
----------
output : str, optional
The path of the output file. The basename of the output file defaults
to the basename of the ``dest_prefix`` value, if supplied; otherwise to
the basename of the environment. The suffix will be determined by the
output format (e.g. ``my_env.tar.gz``).
format : {'infer', 'zip', 'tar.gz', 'tgz', 'tar.bz2', 'tbz2', 'tar', 'parcel', 'squashfs'}
The archival format to use. By default this is inferred from the
output file extension, and defaults to ``tar.gz`` if this is not supplied.
arcroot : str, optional
The relative path in the archive to the conda environment.
Defaults to ''.
dest_prefix : str, optional
If present, prefixes will be rewritten to this path before
packaging. In this case the ``conda-unpack`` script will not be
generated.
parcel_root, parcel_name, parcel_version, parcel_distro : str, optional
(Parcels only) the root directory, name, version, and target distribution
of the parcel. The name and version will be embedded into parcel metadata.
The default values are:
- ``parcel_root``: ``/opt/cloudera/parcels``
- ``parcel_name``: the base name of the environment directory
- ``parcel_version``: the current date in ``YYYY.MM.DD`` format.
- ``parcel_distro``: ``el7``
It is important that ``parcel_root`` match the directory into which all
parcels are unpacked on your cluster. Neither ``parcel_name`` nor
``parcel_version`` may contain dashes. The final destination of the parcel
is assumed to be ``parcel_root/parcel_name-parcel_version``, and both
``arcroot`` and ``dest_prefix`` are set accordingly. The default filename
will be ``parcel_name-parcel_version-parcel_distro.parcel``.
verbose : bool, optional
If True, progress is reported to stdout. Default is False.
force : bool, optional
Whether to overwrite any existing archive at the output path.
Default is False.
compress_level : int, optional
The compression level to use, from 0 to 9. Higher numbers decrease
output file size at the expense of compression time. Ignored for
``format='zip'``. Default is 4.
n_threads : int, optional
The number of threads to use. Set to -1 to use the number of cpus
on this machine. If a file format doesn't support threaded
packaging, this option will be ignored. Default is 1.
zip_symlinks : bool, optional
(``zip`` format only) Symbolic links aren't supported by the Zip standard,
but are supported by *many* common Zip implementations. If ``True``, symbolic
links will be stored in the archive. If ``False``, a copy of the linked file
will be included instead. Choosing ``True`` can avoid storing multiple copies
of the same file, but the archive *may silently fail* on decompression if the
``unzip`` implementation does not support symbolic links. For that reason,
the default is ``False``.
zip_64 : bool, optional
(``zip`` format only) Whether to enable ZIP64 extensions. Default is True.
Returns
-------
out_path : str
The path to the archived environment.
"""
from .formats import archive
# The output path and archive format
output, format = self._output_and_format(output, format)
if format == 'parcel':
if dest_prefix or arcroot:
raise CondaPackException("Cannot specify 'dest_prefix'/'arcroot' for parcels")
dest_prefix, arcroot, parcel = self._parcel_output(parcel_root, parcel_name,
parcel_version, parcel_distro)
if output is None:
output = parcel + '.parcel'
else:
parcel = None
# Ensure the prefix is a relative path
arcroot = arcroot.strip(os.path.sep) if arcroot else ''
if os.path.exists(output) and not force:
raise CondaPackException("File %r already exists" % output)
if verbose:
print(f"Packing environment at {self.prefix!r} to {output!r}")
fd, temp_path = tempfile.mkstemp()
try:
with os.fdopen(fd, 'wb') as temp_file:
with archive(temp_file, temp_path, arcroot, format,
compress_level=compress_level,
zip_symlinks=zip_symlinks,
zip_64=zip_64,
n_threads=n_threads,
verbose=verbose) as arc:
packer = Packer(self.prefix, arc, dest_prefix, parcel)
with progressbar(self.files, enabled=verbose) as files:
for f in files:
packer.add(f)
packer.finish()
except Exception:
# Writing failed, remove tempfile
os.remove(temp_path)
raise
else:
# Writing succeeded, move archive to desired location
shutil.move(temp_path, output)
return output
class File:
"""A single archive record.
Parameters
----------
source : str
Absolute path to the source.
target : str
Relative path from the target prefix (e.g. ``lib/foo/bar.py``).
is_conda : bool, optional
Whether the file was installed by conda, or comes from somewhere else.
file_mode : {None, 'text', 'binary', 'unknown'}, optional
The type of record.
prefix_placeholder : None or str, optional
The prefix placeholder in the file (if any)
"""
__slots__ = ('source', 'target', 'is_conda', 'file_mode',
'prefix_placeholder')
def __init__(self, source, target, is_conda=True, file_mode=None,
prefix_placeholder=None):
self.source = source
self.target = target
self.is_conda = is_conda
self.file_mode = file_mode
self.prefix_placeholder = prefix_placeholder
def __repr__(self):
return f"File<{self.target!r}, is_conda={self.is_conda!r}>"
def pack(name=None, prefix=None, output=None, format='infer',
arcroot='', dest_prefix=None,
parcel_root=None, parcel_name=None,
parcel_version=None, parcel_distro=None,
verbose=False, force=False,
compress_level=4, n_threads=1, zip_symlinks=False, zip_64=True,
filters=None, ignore_editable_packages=False,
ignore_missing_files=False):
"""Package an existing conda environment into an archive file.
Parameters
----------
name : str, optional
The name of the conda environment to pack.
prefix : str, optional
A path to a conda environment to pack.
Only one of ``name`` and ``prefix`` should be supplied.
output : str, optional
The path of the output file. Defaults to the environment name with a
suffix determined by the format; e.g. ``my_env.tar.gz``.
format : {'infer', 'zip', 'tar.gz', 'tgz', 'tar.bz2', 'tbz2', 'tar', 'parcel'}, optional
The archival format to use. By default, this is inferred from the output
file extension, and defaults to ``tar.gz`` if ``output`` is not supplied.
arcroot : str, optional
The relative path in the archive to the conda environment.
Defaults to ''.
dest_prefix : str, optional
If present, prefixes will be rewritten to this path before packaging.
In this case the ``conda-unpack`` script will not be generated.
parcel_root, parcel_name, parcel_version, parcel_distro : str, optional
(Parcels only) the root directory, name, version, and target
distribution of the parcel. The name and version will be embedded
into parcel metadata. The default values are:
- ``parcel_root``: ``/opt/cloudera/parcels``
- ``parcel_name``: the base name of the environment directory
- ``parcel_version``: the current date in ``YYYY.MM.DD`` format.
- ``parcel_distro``: ``el7``
It is important that ``parcel_root`` match the directory into which all
parcels are unpacked on your cluster. Neither ``parcel_name`` nor
``parcel_version`` may contain dashes. The final destination of the parcel
is assumed to be ``parcel_root/parcel_name-parcel_version``, and both
``arcroot`` and ``dest_prefix`` are set accordingly. The default filename
will be ``parcel_name-parcel_version-parcel_distro.parcel``.
verbose : bool, optional
If True, progress is reported to stdout. Default is False.
force : bool, optional
Whether to overwrite any existing archive at the output path. Default
is False.
compress_level : int, optional
The compression level to use, from 0 to 9. Higher numbers decrease
output file size at the expense of compression time. Ignored for
``format='zip'``. Default is 4.
zip_symlinks : bool, optional
(``zip`` format only) Symbolic links aren't supported by the Zip standard,
but are supported by *many* common Zip implementations. If ``True``, symbolic
links will be stored in the archive. If ``False``, a copy of the linked file
will be included instead. Choosing ``True`` can avoid storing multiple copies
of the same file, but the archive *may silently fail* on decompression if the
``unzip`` implementation does not support symbolic links. For that reason,
the default is ``False``.
n_threads : int, optional
The number of threads to use. Set to -1 to use the number of cpus on
this machine. If a file format doesn't support threaded packaging, this
option will be ignored. Default is 1.
zip_64 : bool, optional
(``zip`` format only) Whether to enable ZIP64 extensions. Default is True.
filters : list, optional
A list of filters to apply to the files. Each filter is a tuple of
``(kind, pattern)``, where ``kind`` is either ``'exclude'`` or
``'include'`` and ``pattern`` is a file pattern. Filters are applied in
the order specified.
ignore_editable_packages : bool, optional
By default conda-pack will error in the presence of editable packages.
Set to True to skip these checks.
ignore_missing_files : bool, optional
Ignore that files are missing that should be present in the conda
environment as specified by the conda metadata.
Returns
-------
out_path : str
The path to the archived environment.
"""
if name and prefix:
raise CondaPackException("Cannot specify both 'name' and 'prefix'")
if verbose:
print("Collecting packages...")
if prefix:
env = CondaEnv.from_prefix(prefix,
ignore_editable_packages=ignore_editable_packages,
ignore_missing_files=ignore_missing_files)
elif name:
env = CondaEnv.from_name(name,
ignore_editable_packages=ignore_editable_packages,
ignore_missing_files=ignore_missing_files)
else:
env = CondaEnv.from_default(ignore_editable_packages=ignore_editable_packages,
ignore_missing_files=ignore_missing_files)
if filters is not None:
for kind, pattern in filters:
if kind == 'exclude':
env = env.exclude(pattern)
elif kind == 'include':
env = env.include(pattern)
else:
raise CondaPackException("Unknown filter of kind %r" % kind)
return env.pack(output=output, format=format,
arcroot=arcroot, dest_prefix=dest_prefix,
parcel_root=parcel_root, parcel_name=parcel_name,
parcel_version=parcel_version, parcel_distro=parcel_distro,
verbose=verbose, force=force,
compress_level=compress_level, n_threads=n_threads,
zip_symlinks=zip_symlinks, zip_64=zip_64)
def find_site_packages(prefix):
# Ensure there is at most one version of python installed
pythons = []
for fn in glob.glob(os.path.join(prefix, 'conda-meta', 'python-*.json')):
with open(fn) as fil:
meta = json.load(fil)
if meta['name'] == 'python':
pythons.append(meta)
if len(pythons) > 1: # pragma: nocover
raise CondaPackException("Unexpected failure, multiple versions of "
"python found in prefix %r" % prefix)
elif not pythons:
# No python installed
return None
# Only a single version of python installed in this environment
if on_win:
return 'Lib/site-packages'
python_version = pythons[0]['version']
major_minor = ".".join(python_version.split(".")[:2])
return 'lib/python%s/site-packages' % major_minor
def check_no_editable_packages(prefix, site_packages):
pth_files = glob.glob(os.path.join(prefix, site_packages, '*.pth'))
editable_packages = set()
for pth_fil in pth_files:
dirname = os.path.dirname(pth_fil)
with open(pth_fil) as pth:
for line in pth:
line = line.rstrip()
# Blank lines are skipped
# Lines starting with "#" are skipped
# Lines starting with "import" are executed
if not line or line.startswith('#') or line.startswith('import'):
continue
# All other lines are relative paths
location = os.path.normpath(os.path.join(dirname, line))
if not location.startswith(prefix):
editable_packages.add(line)
if editable_packages:
msg = ("Cannot pack an environment with editable packages\n"
"installed (e.g. from `python setup.py develop` or\n "
"`pip install -e`). Editable packages found:\n\n"
"%s") % '\n'.join('- %s' % p for p in sorted(editable_packages))
raise CondaPackException(msg)
def name_to_prefix(name=None):
try:
conda_exe = os.environ.get("CONDA_EXE", "conda")
info = subprocess.check_output(
f"{conda_exe} info --json", shell=True, stderr=subprocess.PIPE
).decode(default_encoding)
except subprocess.CalledProcessError as exc:
kind = ('current environment' if name is None
else 'environment: %r' % name)
raise CondaPackException("Failed to determine path to %s. This may "
"be due to conda not being on your PATH. The "
"full error is below:\n\n"
"%s" % (kind, exc.output))
info2 = json.loads(info)
if name:
env_lk = {os.path.basename(e): e for e in info2['envs']}
try:
prefix = env_lk[name]
except KeyError:
raise CondaPackException("Environment name %r doesn't exist" % name)
else:
prefix = info2['default_prefix']
return prefix
def read_noarch_type(pkg):
for file_name in ['link.json', 'package_metadata.json']:
path = os.path.join(pkg, 'info', file_name)
if os.path.exists(path):
with open(path) as fil:
info = json.load(fil)
try:
return info['noarch']['type']
except KeyError:
return None
return None
def read_has_prefix(path):
out = {}
with open(path) as fil:
for line in fil:
rec = tuple(x.strip('"\'') for x in shlex.split(line, posix=False))
if len(rec) == 1:
out[rec[0]] = (PREFIX_PLACEHOLDER, 'text')
elif len(rec) == 3:
out[rec[2]] = rec[:2]
else:
raise ValueError("Failed to parse has_prefix file")
return out
def load_files(prefix):
from os.path import isfile, islink, join, relpath
ignore = {'pkgs', 'envs', 'conda-bld', '.conda_lock', 'users',
'conda-recipes', '.index', '.unionfs', '.nonadmin', 'python.app',
'Launcher.app'}
res = set()
for fn in os.listdir(prefix):
if fn in ignore or fn.endswith('~') or fn.endswith('.DS_STORE'):
continue
elif isfile(join(prefix, fn)):
res.add(fn)
elif islink(join(prefix, fn)):
res.add(fn)
else:
for root, dirs, files in os.walk(join(prefix, fn)):
root2 = relpath(root, prefix)
res.update(join(root2, fn2) for fn2 in files)
for d in dirs:
if islink(join(root, d)):
# Add symbolic directory directly
res.add(join(root2, d))
if not dirs and not files:
# root2 is an empty directory, add it
res.add(root2)
return res
def managed_file(is_noarch, site_packages, pkg, _path, prefix_placeholder=None,
file_mode=None, **ignored):
if is_noarch:
if _path.startswith('site-packages/'):
target = site_packages + _path[13:]
elif _path.startswith('python-scripts/'):
target = BIN_DIR + _path[14:]
else:
target = _path
else:
target = _path
return File(os.path.join(pkg, _path),
target,
is_conda=True,
prefix_placeholder=prefix_placeholder,
file_mode=file_mode)
def load_managed_package(info, prefix, site_packages, all_files):
pkg = info['link']['source']
noarch_type = read_noarch_type(pkg)
is_noarch = noarch_type == 'python'
if is_noarch and site_packages is None: # pragma: nocover
raise CondaPackException("noarch: python package installed (%r), but "
"Python not found in environment (%r)" %
(info['name'], prefix))
paths_json = os.path.join(pkg, 'info', 'paths.json')
if os.path.exists(paths_json):
with open(paths_json) as fil:
paths = json.load(fil)
files = [managed_file(is_noarch, site_packages, pkg, **r)
for r in paths['paths']]
else:
with open(os.path.join(pkg, 'info', 'files')) as fil:
paths = [f.strip() for f in fil]
has_prefix = os.path.join(pkg, 'info', 'has_prefix')
if os.path.exists(has_prefix):
prefixes = read_has_prefix(has_prefix)
files = [managed_file(is_noarch, site_packages, pkg, p,
*prefixes.get(p, ())) for p in paths]
else:
files = [managed_file(is_noarch, site_packages, pkg, p)
for p in paths]
if is_noarch:
seen = {os.path.normcase(i.target) for i in files}
for fil in info['files']:
# If the path hasn't been added yet, *and* the path isn't a pyc
# file that failed to bytecode compile (e.g. a file that contains
# py3 only features in a py2 env), then add the path.
fil_normed = os.path.normcase(fil)
if (fil_normed not in seen and not
((fil_normed == ".nonadmin") or
(fil_normed.endswith('.pyc') and fil_normed not in all_files))):
file_mode = 'unknown' if fil.startswith(BIN_DIR) else None
f = File(os.path.join(prefix, fil), fil, is_conda=True,
prefix_placeholder=None, file_mode=file_mode)
files.append(f)
return files
_uncached_error = """
Conda-managed packages were found without entries in the package cache. This
is usually due to `conda clean -p` being unaware of symlinked or copied
packages. Uncached packages:
{0}"""
_uncached_warning = """\
{}
Continuing with packing, treating these packages as if they were unmanaged
files (e.g. from `pip`). This is usually fine, but may cause issues as
prefixes aren't being handled as robustly.""".format(_uncached_error)
_missing_files_error = """
Files managed by conda were found to have been deleted/overwritten in the
following packages:
{0}
This is usually due to `pip` uninstalling or clobbering conda managed files,
resulting in an inconsistent environment. Please check your environment for
conda/pip conflicts using `conda list`, and fix the environment by ensuring
only one version of each package is installed (conda preferred)."""
def load_environment(prefix, on_missing_cache='warn', ignore_editable_packages=False,
ignore_missing_files=False):
# Check if it's a conda environment
if not os.path.exists(prefix):
raise CondaPackException("Environment path %r doesn't exist" % prefix)
conda_meta = os.path.join(prefix, 'conda-meta')
if not os.path.exists(conda_meta):
raise CondaPackException("Path %r is not a conda environment" % prefix)
# Find the environment site_packages (if any)
site_packages = find_site_packages(prefix)
if site_packages is not None and not ignore_editable_packages:
# Check that no editable packages are installed
check_no_editable_packages(prefix, site_packages)
# Save the unnormalized filenames here so that we can preserve the
# case of unmanaged files. The case of managed files is dictated by
# the conda package itself.
all_files = {os.path.normcase(p): p for p in load_files(prefix)}
files = []
managed = set()
uncached = []
missing_files = {}
for path in os.listdir(conda_meta):
if path.endswith('.json'):
with open(os.path.join(conda_meta, path)) as fil:
info = json.load(fil)
pkg = info['link']['source']
if not os.path.exists(pkg):
# Package cache is cleared, set file_mode='unknown' to properly
# handle prefix replacement ourselves later.
new_files = [File(os.path.join(prefix, f), f, is_conda=True,
prefix_placeholder=None, file_mode='unknown')
for f in info['files'] if f != '.nonadmin']
uncached.append((info['name'], info['version'], info['url']))
else:
new_files = load_managed_package(info, prefix, site_packages,
all_files)
targets = {os.path.normcase(f.target) for f in new_files}
new_missing = targets.difference(all_files)
if new_missing:
# Collect packages missing files as we progress to provide a
# complete error message on failure.
missing_files[(info['name'], info['version'])] = new_missing
managed.update(targets)
files.extend(new_files)
# Add conda-meta entry
managed.add(os.path.join('conda-meta', path))
files.append(File(os.path.join(conda_meta, path),
os.path.join('conda-meta', path),
is_conda=True,
prefix_placeholder=None,
file_mode=None))
# Add remaining conda metadata files
managed.add(os.path.join('conda-meta', 'history'))
files.append(File(os.path.join(conda_meta, 'history'),
os.path.join('conda-meta', 'history'),
is_conda=True,
prefix_placeholder=None,
file_mode=None))
if missing_files and not ignore_missing_files:
packages = []
for key, value in missing_files.items():
packages.append('- %s %s:' % key)
value = sorted(value)
if len(value) > 4:
value = value[:3] + ['+ %d others' % (len(value) - 3)]
packages.extend(' ' + p for p in value)
packages = '\n'.join(packages)
raise CondaPackException(_missing_files_error.format(packages))
# Add unmanaged files, preserving their original case
unmanaged = {fn for fn_l, fn in all_files.items() if fn_l not in managed}
# Older versions of conda insert unmanaged conda, activate, and deactivate
# scripts into child environments upon activation. Remove these
fnames = ('conda', 'activate', 'deactivate')
if on_win:
# Windows includes the POSIX and .bat versions of each
fnames = fnames + ('conda.bat', 'activate.bat', 'deactivate.bat')
unmanaged -= {os.path.join(BIN_DIR, f) for f in fnames}
files.extend(File(os.path.join(prefix, p),
p,
is_conda=False,
prefix_placeholder=None,
file_mode='unknown')
for p in unmanaged if not find_py_source(p) in managed)
if uncached and on_missing_cache in ('warn', 'raise'):
packages = '\n'.join('- %s=%r %s' % i for i in uncached)
if on_missing_cache == 'warn':
context.warn(_uncached_warning.format(packages))
else:
raise CondaPackException(_uncached_error.format(packages))
return files
def rewrite_shebang(data, target, prefix):
"""Rewrite a shebang header to ``#!usr/bin/env program...``.
Returns
-------
data : bytes
fixed : bool
Whether the file was successfully fixed in the rewrite.
"""
shebang_match = re.match(SHEBANG_REGEX, data, re.MULTILINE)
prefix_b = prefix.encode('utf-8')
if shebang_match:
if data.count(prefix_b) > 1:
# More than one occurrence of prefix, can't fully cleanup.
return data, False
shebang, executable, options = shebang_match.groups()
if executable.startswith(prefix_b):
# shebang points inside environment, rewrite
executable_name = executable.decode("utf-8").split("/")[-1]
new_shebang = "#!/usr/bin/env {}{}".format(
executable_name, options.decode("utf-8")
)
data = data.replace(shebang, new_shebang.encode("utf-8"))
return data, True
return data, False
def rewrite_conda_meta(source):
"""Remove absolute paths in conda-meta that reference local install.
These are unnecessary for install/uninstall on the destination machine."""
with open(source) as f:
original = f.read()
data = json.loads(original)
for field in ["extracted_package_dir", "package_tarball_full_path"]:
if field in data:
data[field] = ""
if "link" in data and "source" in data["link"]:
data["link"]["source"] = ""
out = json.dumps(data, indent=True, sort_keys=True)
return out.encode(), data
_parcel_json_template = """\
{{
"components": [
{{
"name": "{parcel_name}",
"pkg_version": "{parcel_version}",
"version": "{parcel_version}"
}}
],
"extraVersionInfo": {{
"baseVersion": "{parcel_version}",
"fullVersion": "{parcel_version}-{parcel_distro}",
"patchCount": "0"
}},
"groups": [],
"name": "{parcel_name}",
"packages": [
{parcel_packages}
],
"provides": [
"spark-plugin"
],
"schema_version": 1,
"scripts": {{
"defines": "conda_env.sh"
}},
"setActiveSymlink": true,
"users": {{}},
"version": "{parcel_version}"
}}"""
_parcel_package_template = """\
{{
"name": "{name}",
"version": "{version}-{build}"
}}"""
_conda_unpack_template = """\
{shebang}
{prefixes_py}
_prefix_records = [
{prefix_records}
]
if __name__ == '__main__':
import os
import argparse
parser = argparse.ArgumentParser(
prog='conda-unpack',
description=('Finish unpacking the environment after unarchiving. '
'Cleans up absolute prefixes in any remaining files'))
parser.add_argument('--version',
action='store_true',
help='Show version then exit')
args = parser.parse_args()
# Manually handle version printing to output to stdout in python < 3.4
if args.version:
print('conda-unpack {version}')
else:
script_dir = os.path.dirname(__file__)
new_prefix = os.path.abspath(os.path.dirname(script_dir))
for path, placeholder, mode in _prefix_records:
new_path = os.path.join(new_prefix, path)
if on_win:
new_path = new_path.replace('\\\\', '/')
update_prefix(new_path, new_prefix, placeholder, mode=mode)
"""
# Deduce file type for unmanaged packages. If decoding utf-8 is
# successful, we assume text, otherwise binary.
def is_binary_file(data):
try:
data.decode('utf-8')
return False
except UnicodeDecodeError:
return True
class Packer:
def __init__(self, prefix, archive, dest_prefix=None, parcel=None):
self.prefix = prefix
self.archive = archive
self.dest = dest_prefix
self.has_dest = dest_prefix is not None
self.parcel = parcel
self.prefixes = []
self.packages = []
def add(self, file):
# Windows note:
# When adding files to an archive, that archive is generally
# case-sensitive. The target paths can be mixed case, and that means
# that they will be distinct directories in the archive.
#
# If those files are then extracted onto a case-sensitive file-system
# (such as a network share), Windows will not be able to traverse them
# correctly.
#
# The simple (undesirable) solution is to normalize (lowercase) the
# filenames when adding them to the archive.
#
# A nicer solution would be to note the "canonical" capitalization of a
# prefix when it first occurs, and use that every time the prefix
# occurs subsequently.
#
# We just ignore this problem for the time being.
if file.file_mode is None:
if fnmatch(file.target, 'conda-meta/*.json'):
# Detect if conda is installed
out, data = rewrite_conda_meta(file.source)
self.packages.append(data)
self.archive.add_bytes(file.source, out, file.target)
else:
self.archive.add(file.source, file.target)
return
elif file.file_mode not in ('text', 'binary', 'unknown'):
raise ValueError("unknown file_mode: %r" % file.file_mode) # pragma: no cover
elif os.path.isdir(file.source) or os.path.islink(file.source):
self.archive.add(file.source, file.target)
return
file_mode = file.file_mode
placeholder = file.prefix_placeholder
if (
self.has_dest
or file_mode == "unknown"
or file_mode == "text"
and file.target.startswith(BIN_DIR)
):
# In each of these cases, we need to inspect the file contents here.
with open(file.source, 'rb') as fil:
data = fil.read()
else:
# No need to read the file; just pass the filename to the archiver.
self.archive.add(file.source, file.target)
self.prefixes.append((file.target, placeholder, file_mode))
return
if file_mode == 'unknown':
placeholder = self.prefix
if is_binary_file(data):
if on_win:
# The only binary replacement we do on Windows is distlib
# shebang replacement, safe for unmanaged binaries. For
# Unix (and other Windows binaries), we cannot trust that
# binary replacement can be done safely.
file_mode = 'binary'
else:
file_mode = 'text'
if file_mode != 'unknown':
if self.has_dest:
data = replace_prefix(data, file_mode, placeholder, self.dest)
else:
if file_mode == 'text' and file.target.startswith(BIN_DIR):
data, fixed = rewrite_shebang(data, file.target, placeholder)
else:
fixed = False
if not fixed:
self.prefixes.append((file.target, placeholder, file_mode))
self.archive.add_bytes(file.source, data, file.target)
def _write_text_file(self, fpath, ftext, executable=False):
fil = tempfile.NamedTemporaryFile(mode='w', delete=False)
try:
fil.write(ftext)
fil.close()
st = os.stat(fil.name)
if executable:
os.chmod(fil.name, st.st_mode | 0o111)
self.archive.add(fil.name, fpath)
finally:
os.unlink(fil.name)
def finish(self):
from . import __version__ # local import to avoid circular imports
from .formats import SquashFSArchive
# Parcel mode
if self.parcel:
src = os.path.join(_current_dir, 'scripts', 'posix', 'parcel')
dst = os.path.join('meta', 'conda_env.sh')
self.archive.add(src, dst)
parcel_name, parcel_vd = self.parcel.split('-', 1)
parcel_version, parcel_distro = parcel_vd.rsplit('-', 1)
parcel_packages = ',\n'.join(_parcel_package_template.format(**p)
for p in self.packages)
parcel_json = _parcel_json_template.format(parcel_name=parcel_name,
parcel_version=parcel_version,
parcel_packages=parcel_packages,
parcel_distro=parcel_distro)
dst = os.path.join('meta', 'parcel.json')
self._write_text_file(dst, parcel_json, False)
return
# Add conda-pack's activate/deactivate scripts
has_conda = any(d['name'] == 'conda' for d in self.packages)
if not (self.has_dest and has_conda):
for source, target in _scripts:
self.archive.add(source, target)
# No `conda-unpack` command if dest-prefix specified
if self.has_dest:
return
if on_win:
shebang = '#!python.exe'
# Don't just use os.path.join here: the backslash needs
# to be doubled up for the sake of the regex match
python_pattern = re.compile(BIN_DIR + r'\\python\d.\d')
else:
shebang = '#!/usr/bin/env python'
python_pattern = re.compile(BIN_DIR + '/python')
# We skip prefix rewriting in python executables (if needed)
# to avoid editing a running file.
prefix_records = ',\n'.join(repr(p) for p in self.prefixes
if not python_pattern.match(p[0]))
with open(os.path.join(_current_dir, 'prefixes.py')) as fil:
prefixes_py = fil.read()
script = _conda_unpack_template.format(shebang=shebang,
prefix_records=prefix_records,
prefixes_py=prefixes_py,
version=__version__)
script_name = 'conda-unpack-script.py' if on_win else 'conda-unpack'
self._write_text_file(os.path.join(BIN_DIR, script_name), script, True)
if on_win:
exe = 'cli-32.exe' if is_32bit else 'cli-64.exe'
cli_exe = pkg_resources.resource_filename('setuptools', exe)
self.archive.add(cli_exe, os.path.join(BIN_DIR, 'conda-unpack.exe'))
# mksquashfs has no (fast) iterative mode, only batch mode
# therefore can do the actual squashing only once we've added all the files
if isinstance(self.archive, SquashFSArchive):
self.archive.mksquashfs_from_staging()