Commit a4b4fc3d authored by Daniel P. Berrange's avatar Daniel P. Berrange

docker: introduce a DockerRegistry class

Introduce a class to handle HTTP requests with a docker registry
server, and associated auth credentials.
Signed-off-by: 's avatarDaniel P. Berrange <berrange@redhat.com>
parent b20deaac
......@@ -31,6 +31,7 @@ import shutil
import urlparse
import hashlib
from abc import ABCMeta, abstractmethod
import copy
from . import base
......@@ -152,14 +153,132 @@ class DockerAuthToken(DockerAuth):
return False
class DockerSource(base.Source):
class DockerRegistry():
def __init__(self):
def __init__(self, uri_base):
self.uri_base = list(urlparse.urlparse(uri_base))
self.auth_handler = DockerAuthNop()
def set_auth_handler(self, auth_handler):
self.auth_handler = auth_handler
def set_server(self, server):
self.uri_base[1] = server
@classmethod
def from_template(cls, template):
protocol = template.protocol
hostname = template.hostname
port = template.port
if protocol is None:
protocol = "https"
if hostname is None:
hostname = "index.docker.io"
if port is None:
server = hostname
else:
server = "%s:%s" % (hostname, port)
url = urlparse.urlunparse((protocol, server, "", None, None, None))
return cls(url)
def get_url(self, path, headers=None):
url_bits = copy.copy(self.uri_base)
url_bits[2] = path
url = urlparse.urlunparse(url_bits)
debug("Fetching %s..." % url)
req = urllib2.Request(url=url)
if headers is not None:
for h in headers.keys():
req.add_header(h, headers[h])
self.auth_handler.prepare_req(req)
try:
res = urllib2.urlopen(req)
self.auth_handler.process_res(res)
return res
except urllib2.HTTPError as e:
if e.code == 401:
retry = self.auth_handler.process_err(e)
if retry:
debug("Re-Fetching %s..." % url)
self.auth_handler.prepare_req(req)
res = urllib2.urlopen(req)
self.auth_handler.process_res(res)
return res
else:
debug("Not re-fetching")
raise
else:
raise
def save_data(self, path, dest, checksum=None):
try:
res = self.get_url(path)
datalen = res.info().getheader("Content-Length")
if datalen is not None:
datalen = int(datalen)
csum = None
if checksum is not None:
csum = hashlib.sha256()
pattern = [".", "o", "O", "o"]
patternIndex = 0
donelen = 0
with open(dest, "w") as f:
while 1:
buf = res.read(1024*64)
if not buf:
break
if csum is not None:
csum.update(buf)
f.write(buf)
if datalen is not None:
donelen = donelen + len(buf)
debug("\x1b[s%s (%5d Kb of %5d Kb)\x1b8" % (
pattern[patternIndex], (donelen/1024), (datalen/1024)
))
patternIndex = (patternIndex + 1) % 4
debug("\x1b[K")
if csum is not None:
csumstr = "sha256:" + csum.hexdigest()
if csumstr != checksum:
debug("FAIL checksum '%s' does not match '%s'" % (csumstr, checksum))
os.remove(dest)
raise IOError("Checksum '%s' for data does not match '%s'" % (csumstr, checksum))
debug("OK\n")
return res
except Exception, e:
debug("FAIL %s\n" % str(e))
raise
def get_json(self, path):
try:
headers = {}
headers["Accept"] = "application/json"
res = self.get_url(path, headers)
data = json.loads(res.read())
debug("OK\n")
return (data, res)
except Exception, e:
debug("FAIL %s\n" % str(e))
raise
class DockerSource(base.Source):
def _check_cert_validate(self):
major = sys.version_info.major
SSL_WARNING = "SSL certificates couldn't be validated by default. You need to have 2.7.9/3.4.3 or higher"
......@@ -189,38 +308,33 @@ class DockerSource(base.Source):
def download_template(self, image, template, templatedir):
self._check_cert_validate()
registry = DockerRegistry.from_template(template)
basicauth = DockerAuthBasic(template.username, template.password)
self.set_auth_handler(basicauth)
registry.set_auth_handler(basicauth)
try:
(data, res) = self._get_json(template,
None,
"/v1/repositories/%s/%s/images" % (
image.repo, image.name,
))
(data, res) = registry.get_json("/v1/repositories/%s/%s/images" % (
image.repo, image.name,
))
except urllib2.HTTPError, e:
raise ValueError(["Image '%s' does not exist" % template])
registryendpoint = res.info().getheader('X-Docker-Endpoints')
if basicauth.token is not None:
self.set_auth_handler(DockerAuthToken(basicauth.token))
registry.set_auth_handler(DockerAuthToken(basicauth.token))
else:
self.set_auth_handler(DockerAuthNop())
registry.set_auth_handler(DockerAuthNop())
(data, res) = self._get_json(template,
registryendpoint,
"/v1/repositories/%s/%s/tags" %(
image.repo, image.name
))
(data, res) = registry.get_json("/v1/repositories/%s/%s/tags" %(
image.repo, image.name
))
if image.tag not in data:
raise ValueError(["Tag '%s' does not exist for image '%s'" %
(image.tag, template)])
imagetagid = data[image.tag]
(data, res) = self._get_json(template,
registryendpoint,
"/v1/images/" + imagetagid + "/ancestry")
(data, res) = registry.get_json("/v1/images/" + imagetagid + "/ancestry")
if data[0] != imagetagid:
raise ValueError(["Expected first layer id '%s' to match image id '%s'",
......@@ -240,16 +354,12 @@ class DockerSource(base.Source):
datafile = layerdir + "/template.tar.gz"
if not os.path.exists(jsonfile) or not os.path.exists(datafile):
res = self._save_data(template,
registryendpoint,
"/v1/images/" + layerid + "/json",
jsonfile)
res = registry.save_data("/v1/images/" + layerid + "/json",
jsonfile)
createdFiles.append(jsonfile)
self._save_data(template,
registryendpoint,
"/v1/images/" + layerid + "/layer",
datafile)
registry.save_data("/v1/images/" + layerid + "/layer",
datafile)
createdFiles.append(datafile)
index = {
......@@ -275,108 +385,6 @@ class DockerSource(base.Source):
except:
pass
def _save_data(self, template, server, path,
dest, checksum=None):
try:
res = self._get_url(template, server, path)
datalen = res.info().getheader("Content-Length")
if datalen is not None:
datalen = int(datalen)
csum = None
if checksum is not None:
csum = hashlib.sha256()
pattern = [".", "o", "O", "o"]
patternIndex = 0
donelen = 0
with open(dest, "w") as f:
while 1:
buf = res.read(1024*64)
if not buf:
break
if csum is not None:
csum.update(buf)
f.write(buf)
if datalen is not None:
donelen = donelen + len(buf)
debug("\x1b[s%s (%5d Kb of %5d Kb)\x1b8" % (
pattern[patternIndex], (donelen/1024), (datalen/1024)
))
patternIndex = (patternIndex + 1) % 4
debug("\x1b[K")
if csum is not None:
csumstr = "sha256:" + csum.hexdigest()
if csumstr != checksum:
debug("FAIL checksum '%s' does not match '%s'" % (csumstr, checksum))
os.remove(dest)
raise IOError("Checksum '%s' for data does not match '%s'" % (csumstr, checksum))
debug("OK\n")
return res
except Exception, e:
debug("FAIL %s\n" % str(e))
raise
def _get_url(self, template, server, path, headers=None):
if template.protocol is None:
protocol = "https"
else:
protocol = template.protocol
if server is None:
if template.hostname is None:
server = "index.docker.io"
else:
if template.port is not None:
server = "%s:%d" % (template.hostname, template.port)
else:
server = template.hostname
url = urlparse.urlunparse((protocol, server, path, None, None, None))
debug("Fetching %s..." % url)
req = urllib2.Request(url=url)
if headers is not None:
for h in headers.keys():
req.add_header(h, headers[h])
self.auth_handler.prepare_req(req)
try:
res = urllib2.urlopen(req)
self.auth_handler.process_res(res)
return res
except urllib2.HTTPError as e:
if e.code == 401:
retry = self.auth_handler.process_err(e)
if retry:
debug("Re-Fetching %s..." % url)
self.auth_handler.prepare_req(req)
res = urllib2.urlopen(req)
self.auth_handler.process_res(res)
return res
else:
debug("Not re-fetching")
raise
else:
raise
def _get_json(self, template, server, path):
try:
headers = {}
headers["Accept"] = "application/json")
res = self._get_url(template, server, path, headers)
data = json.loads(res.read())
debug("OK\n")
return (data, res)
except Exception, e:
debug("FAIL %s\n" % str(e))
raise
def create_template(self, template, templatedir, connect=None):
image = DockerImage.from_template(template)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment