File: C:/Users/fred/anaconda3/Lib/site-packages/anaconda_project/client.py
# -*- coding: utf-8 -*-
# -----------------------------------------------------------------------------
# Copyright (c) 2016, Anaconda, Inc. All rights reserved.
#
# Licensed under the terms of the BSD 3-Clause License.
# The full license is in the file LICENSE.txt, distributed with this software.
# -----------------------------------------------------------------------------
"""Talking to the Anaconda server."""
from __future__ import absolute_import, print_function
import logging
import os
import re
import tarfile
import zipfile
from tqdm import tqdm
import requests
import binstar_client.utils as binstar_utils
import binstar_client.requests_ext as binstar_requests_ext
from binstar_client.errors import BinstarError, Unauthorized
from anaconda_project.internal.simple_status import SimpleStatus
def _basename(fname):
base1, ext1 = os.path.splitext(fname)
if ext1 == '.zip':
return base1
elif ext1 in ['.gz', '.bz2']:
base2, ext1 = os.path.splitext(base1)
return base2
else:
raise ValueError('{} does not appear to be a compressed archive.'.format(fname))
class _Client(object):
def __init__(self, site=None, username=None, token=None, log_level=None):
assert hasattr(binstar_utils, 'get_server_api'), "Please upgrade anaconda-client"
if log_level is None:
log_level = logging.INFO
self._api = binstar_utils.get_server_api(site=site, token=token, log_level=log_level)
self._user_info = None
self._force_username = username
def _username(self):
"""Get username to upload to; raise Unauthorized if we aren't logged in."""
if self._user_info is None:
self._user_info = self._api.user(login=self._force_username)
assert self._user_info is not None
username = self._user_info.get('login', None)
if username is None:
raise Unauthorized()
else:
return username
def _check_response(self, res, allowed=(200, 201)):
# using a little private API here.
self._api._check_response(res, allowed=list(allowed))
# HACK ALERT: using this is a workaround for
# https://github.com/Anaconda-Platform/anaconda-server/issues/2229
def _exists(self, project_name, username=None):
if username:
url = "{}/apps/{}/projects/{}".format(self._api.domain, username, project_name)
else:
url = "{}/apps/{}/projects/{}".format(self._api.domain, self._username(), project_name)
res = self._api.session.get(url)
return res.status_code == 200
def create(self, project_info, private):
url = "{}/apps/{}/projects".format(self._api.domain, self._username())
json = {
'name': project_info['name'],
'access': 'public',
'profile': {
'description': project_info['description']
}
}
if private:
json['access'] = 'private'
data, headers = binstar_utils.jencode(json)
res = self._api.session.post(url, data=data, headers=headers)
self._check_response(res)
return res
def _file_count(self, archive_filename):
for suffix in (".tar", ".tar.gz", ".tar.bz2"):
if archive_filename.lower().endswith(suffix):
with tarfile.open(archive_filename, 'r') as tf:
return len(tf.getnames())
if archive_filename.lower().endswith(".zip"):
with zipfile.ZipFile(archive_filename, 'r') as zf:
return len(zf.namelist())
assert False, ("unsupported archive filename %s" % archive_filename) # pragma: no cover (should not be reached)
def stage(self, project_info, archive_filename, uploaded_basename, private):
url = "{}/apps/{}/projects/{}/stage".format(self._api.domain, self._username(), project_info['name'])
config = project_info.copy()
config['size'] = os.path.getsize(archive_filename)
if private:
config['access'] = 'private'
file_count = self._file_count(archive_filename)
if file_count is not None:
config['num_of_files'] = file_count
json = {'basename': uploaded_basename, 'configuration': config}
data, headers = binstar_utils.jencode(json)
res = self._api.session.post(url, data=data, headers=headers)
self._check_response(res)
return res
def commit(self, project_name, dist_id):
url = "{}/apps/{}/projects/{}/commit/{}".format(self._api.domain, self._username(), project_name, dist_id)
data, headers = binstar_utils.jencode({})
res = self._api.session.post(url, data=data, headers=headers)
self._check_response(res)
return res
def _put_on_s3(self, archive_filename, uploaded_basename, url, s3data):
with open(archive_filename, 'rb') as f:
_hexmd5, b64md5, size = binstar_utils.compute_hash(f, size=os.path.getsize(archive_filename))
s3data = s3data.copy() # don't modify our parameters
s3data['Content-Length'] = size
s3data['Content-MD5'] = b64md5
with open(archive_filename, 'rb') as archive_file_object:
data_stream, headers = binstar_requests_ext.stream_multipart(
s3data, files={'file': (uploaded_basename, archive_file_object)})
res = requests.post(url,
data=data_stream,
verify=self._api.session.verify,
timeout=10 * 60 * 60,
headers=headers)
self._check_response(res)
return res
def upload(self, project_info, archive_filename, uploaded_basename, private):
"""Upload archive_filename created from project, throwing BinstarError."""
if not self._exists(project_info['name']):
res = self.create(project_info=project_info, private=private)
assert res.status_code in (200, 201)
res = self.stage(project_info=project_info,
archive_filename=archive_filename,
uploaded_basename=uploaded_basename,
private=private)
assert res.status_code in (200, 201)
stage_info = res.json()
assert 'post_url' in stage_info
assert 'form_data' in stage_info
assert 'dist_id' in stage_info
res = self._put_on_s3(archive_filename,
uploaded_basename,
url=stage_info['post_url'],
s3data=stage_info['form_data'])
assert res.status_code in (200, 201)
res = self.commit(project_info['name'], stage_info['dist_id'])
assert res.status_code in (200, 201)
return res.json()
def download(self, project, project_dir=None, parent_dir=None):
"""Download project archive and extract."""
if '/' in project:
owner, project_name = project.split('/')
else:
owner = self._username()
project_name = project
if not self._exists(project_name, owner):
raise BinstarError('404')
url = "{}/apps/{}/projects/{}/download".format(self._api.domain, owner, project_name)
data, headers = binstar_utils.jencode({})
with self._api.session.get(url, data=data, headers=headers, stream=True) as res:
res.raise_for_status()
filename = eval(re.findall("filename=(.+);", res.headers["Content-Disposition"])[0])
if parent_dir:
filename = os.path.join(parent_dir, filename)
progress = tqdm(unit='KiB',
total=int(res.headers.get('Content-Length', None)) / 1024,
unit_scale=True,
desc='Download')
with open(filename, 'wb') as f:
for chunk in res.iter_content(chunk_size=1024):
if chunk:
progress.update(len(chunk) / 1024)
f.write(chunk)
progress.close()
self._check_response(res)
return os.path.abspath(filename)
class _UploadedStatus(SimpleStatus):
def __init__(self, json):
self.url = json.get('url', None)
msg = 'Upload successful' + ('.' if self.url is None else ': ' + self.url)
super(_UploadedStatus, self).__init__(success=True, description=msg)
class _DownloadedStatus(SimpleStatus):
def __init__(self, filename):
self.filename = filename
msg = 'Download successful' + ('.' if filename is None else ': ' + filename)
super(_DownloadedStatus, self).__init__(success=True, description=msg)
# This function is supposed to encapsulate the binstar API (don't
# require any other files to import binstar_client).
# archive_filename is the path to a local tmp file to upload
# uploaded_basename is the filename the server should remember
def _upload(project,
archive_filename,
uploaded_basename,
private=None,
site=None,
username=None,
token=None,
log_level=None):
assert not project.problems
client = _Client(site=site, username=username, token=token, log_level=log_level)
try:
json = client.upload(project.publication_info(), archive_filename, uploaded_basename, private)
return _UploadedStatus(json)
except Unauthorized:
return SimpleStatus(success=False,
description='Please log in with the "anaconda login" command.',
errors=["Not logged in."])
except BinstarError as e:
return SimpleStatus(success=False, description="Upload failed.", errors=[str(e)])
def _download(project, project_dir=None, parent_dir=None, site=None, username=None, token=None, log_level=None):
client = _Client(site=site, username=username, token=token, log_level=log_level)
try:
fn = client.download(project, project_dir, parent_dir)
return _DownloadedStatus(fn)
except BinstarError as e:
return SimpleStatus(success=False, description="{} was not found.".format(project), errors=[str(e)])