diff options
author | 2017-08-14 07:05:18 +0300 | |
---|---|---|
committer | 2017-08-14 07:05:18 +0300 | |
commit | c079109cc3681fbfcc95def343435fe65bb1f9b9 (patch) | |
tree | c20dad21f50e3be2225bd0a2caab8f2e5e210841 | |
parent | Added a RemoteRepo object for remote fetching (diff) | |
download | pomu-c079109cc3681fbfcc95def343435fe65bb1f9b9.tar.gz pomu-c079109cc3681fbfcc95def343435fe65bb1f9b9.tar.bz2 pomu-c079109cc3681fbfcc95def343435fe65bb1f9b9.zip |
Remote fetching
Added classes to fetch packages/trees etc. from remote repositories of
all kinds mentioned in overlays.xml: git, mercurial, svn, rsync
Added utility functions for git repos and to interact with filesystem
repository hierarchy
-rw-r--r-- | pomu/repo/remote/git.py | 68 | ||||
-rw-r--r-- | pomu/repo/remote/hg.py | 58 | ||||
-rw-r--r-- | pomu/repo/remote/remote.py | 23 | ||||
-rw-r--r-- | pomu/repo/remote/rsync.py | 59 | ||||
-rw-r--r-- | pomu/repo/remote/svn.py | 49 | ||||
-rw-r--r-- | pomu/util/git.py | 22 | ||||
-rw-r--r-- | pomu/util/remote.py | 38 |
7 files changed, 304 insertions, 13 deletions
diff --git a/pomu/repo/remote/git.py b/pomu/repo/remote/git.py new file mode 100644 index 0000000..217a0c6 --- /dev/null +++ b/pomu/repo/remote/git.py @@ -0,0 +1,68 @@ +"""A class for remote git repos""" +from os import chdir, mkdtemp, path +from subprocess import call + +from git import Repo + +from pomu.repo.remote import RemoteRepo, normalize_key +from pomu.util.git import parse_object +from pomu.util.result import Result + +class RemoteGitRepo(RemoteRepo): + """A class responsible for git remotes""" + def __init__(self, url): + self.uri = uri + self.dir = mkdtemp() + chdir(self.dir) + if call('git', 'clone', '--depth=1', '--bare', url) > 0: # we've a problem + raise RuntimeError() + self.repo = Repo(self.dir) + + def __enter__(self): + pass + + def __exit__(self, *_): + self.cleanup() + + def get_object(self, oid): + head, tail = oid[0:2], oid[2:] + opath = path.join(self.dir, 'objects', head, tail) + return a.read() + + def _fetch_tree(self, obj, tpath): + res = [] + ents = parse_object(self.get_object(tid), tpath).unwrap() + for is_dir, sha, opath in ents: + res.append((opath + '/' if is_dir else '', sha)) + if is_dir: + res.extend(self._fetch_tree(sha, tpath + opath)) + return res + + def fetch_tree(self): + """Returns repos hierarchy""" + if hasattr(self, '_tree'): + return self._tree + tid = self.repo.tree().hexsha + res = self._fetch_tree(tid, '') + self._tree = res + return [x for x, y in res] + + def fetch_subtree(self, key): + """Lists a subtree""" + k = normalize_key(key, True) + self.fetch_tree() + dic = dict(self._tree) + if k not in dic: + return Result.Err() + l = len(key) + return Result.Ok( + [tpath[l:] for tpath in self.fetch_tree() if tpath.startswith(k)]) + + def fetch_file(self, key): + """Fetches a file from the repo""" + k = normalize_key(key) + self.fetch_tree() + dic = dict(self._tree) + if k not in dic: + return Result.Err() + return Result.Ok(parse_object(self.get_object(dic[k]))) diff --git a/pomu/repo/remote/hg.py b/pomu/repo/remote/hg.py new file mode 100644 index 0000000..678f843 --- /dev/null +++ b/pomu/repo/remote/hg.py @@ -0,0 +1,58 @@ +"""A class for remote hg repos""" +from os import chdir, mkdtemp, path +from shutil import rmtree +from subprocess import call, run, check_output + +from pomu.repo.remote import RemoteRepo, normalize_key +from pomu.util.result import Result + +class RemoteHgRepo(RemoteRepo): + """A class responsible for hg remotes""" + def __init__(self, url): + self.uri = uri + self.dir = mkdtemp() + chdir(self.dir) + if call('hg', 'clone', '-U', url, '.') > 0: # we've a problem + raise RuntimeError() + + def __enter__(self): + pass + + def __exit__(self, *_): + self.cleanup() + + def fetch_tree(self): + """Returns repos hierarchy""" + if hasattr(self, '_tree'): + return self._tree + p = run('hg', 'files', '-rdefault') + if p.returncode: + return [] + self._tree = ['/' + x for x in p.stdout.split('\n')] + return self._tree + + def fetch_subtree(self, key): + """Lists a subtree""" + k = normalize_key(key, True) + self.fetch_tree() + dic = dict(self._tree) + if k not in dic: + return Result.Err() + l = len(key) + return Result.Ok( + [tpath[l:] for tpath in self.fetch_tree() if tpath.startswith(k)]) + + def fetch_file(self, key): + """Fetches a file from the repo""" + k = normalize_key(key) + self.fetch_tree() + dic = dict(self._tree) + if k not in dic: + return Result.Err() + p = run('hg', 'cat', '-rdefault', k) + if p.returncode: + return Result.Err() + return Result.Ok(p.stdout) + + def cleanup(self): + rmtree(self.dir) diff --git a/pomu/repo/remote/remote.py b/pomu/repo/remote/remote.py index e177f68..d057e6f 100644 --- a/pomu/repo/remote/remote.py +++ b/pomu/repo/remote/remote.py @@ -4,16 +4,28 @@ class RemoteRepo(): """A class responsible for remotes""" def __init__(self, url): - self.uri = uri raise NotImplementedError() def fetch_package(self, name, category=None, version=None): """Fetches a package, determined by the parametres""" - raise NotImplementedError() + cat, n, ver = get_full_cpv(self.fetch_tree(), name, category, version).unwrap() + ebuild = '{}/{}/{}-{}.ebuild'.format(cat, n, n, ver) + subdir = '/{}/{}'.format(category, name) + filemap = {} + filemap[ebuild] = self.fetch_file(ebuild).unwrap() + subtree = self.fetch_subtree('/{}/{}/'.format(category, name)) + for fpath in subtree: + if '/' in fpath: + parent, _, child = fpath.rpartition('/') + if parent != 'files': continue + if fpath.endswith('.ebuild') or fpath.endswith('/'): continue + p = path.join(subdir, fpath) + filemap[p] = self.fetch_file(p).unwrap() + return Package(name, '/', None, category, version, filemap=filemap) def list_cpvs(self): """Gets a list of all pebuilds in the repo""" - raise NotImplementedError() + return filelist_to_cpvs(self.fetch_tree()) def fetch_tree(self): """Returns repos hierarchy""" @@ -26,3 +38,8 @@ class RemoteRepo(): def fetch_file(self, key): """Fetches a file from the repo""" raise NotImplementedError() + +def normalize_key(key, trail=False): + k = '/' + key.lstrip('/') + if trail: + k = k.rstrip('/') + '/' diff --git a/pomu/repo/remote/rsync.py b/pomu/repo/remote/rsync.py new file mode 100644 index 0000000..8f00e5e --- /dev/null +++ b/pomu/repo/remote/rsync.py @@ -0,0 +1,59 @@ +"""A class for remote rsync repos""" +from os import mkdtemp, rmdir, mkfifo, unlink +import re +from subprocess import run + +from pomu.repo.remote import RemoteRepo, normalize_key +from pomu.util.result import Result + +class RemoteRsyncRepo(RemoteRepo): + """A class responsible for rsync remotes""" + def __init__(self, url): + self.uri = uri + + def __enter__(self): + pass + + def __exit__(self, *_): + pass + + def fetch_tree(self): + """Returns repos hierarchy""" + if hasattr(self, '_tree'): + return self._tree + d = mkdtemp() + p = run('rsync', '-rn', '--out-format="%n"' self.uri, d) + rmdir(d) + if p.returncode: + return Result.Err() + self._tree = ['/' + x for x in p.stdout.split('\n')] + return self._tree + + def fetch_subtree(self, key): + """Lists a subtree""" + k = normalize_key(key, True) + self.fetch_tree() + dic = dict(self._tree) + if k not in dic: + return [] + l = len(key) + return Result.Ok( + [tpath[l:] for tpath in self.fetch_tree() if tpath.startswith(k)]) + + def fetch_file(self, key): + """Fetches a file from the repo""" + k = normalize_key(key) + self.fetch_tree() + dic = dict(self._tree) + if k not in dic: + return Result.Err() + d = mkdtemp() + fip = path.join(d, 'fifo') + mkfifo(fip) + p = run('rsync', self.uri.rstrip('/') + key, fip) + fout = fip.read() + unlink(fip) + rmdir(d) + if p.returncode: + return Result.Err() + return Result.Ok(fout) diff --git a/pomu/repo/remote/svn.py b/pomu/repo/remote/svn.py new file mode 100644 index 0000000..4b1e1d3 --- /dev/null +++ b/pomu/repo/remote/svn.py @@ -0,0 +1,49 @@ +"""A class for remote svn repos""" +from subprocess import run + +from pomu.repo.remote import RemoteRepo, normalize_key +from pomu.util.result import Result + +class RemoteSvnRepo(RemoteRepo): + """A class responsible for svn remotes""" + def __init__(self, url): + self.uri = uri + + def __enter__(self): + pass + + def __exit__(self, *_): + pass + + def fetch_tree(self): + """Returns repos hierarchy""" + if hasattr(self, '_tree'): + return self._tree + p = run('svn', 'ls', '-R', self.uri) + if p.returncode: + return [] + self._tree = p.stdout.split('\n') + return self._tree + + def fetch_subtree(self, key): + """Lists a subtree""" + k = normalize_key(key, True) + self.fetch_tree() + dic = dict(self._tree) + if k not in dic: + return Result.Err() + l = len(key) + return Result.Ok( + [tpath[l:] for tpath in self.fetch_tree() if tpath.startswith(k)]) + + def fetch_file(self, key): + """Fetches a file from the repo""" + k = normalize_key(key) + self.fetch_tree() + dic = dict(self._tree) + if k not in dic: + return Result.Err() + p = run('svn', 'cat', k) + if p.returncode: + return Result.Err() + return Result.Ok(p.stdout) diff --git a/pomu/util/git.py b/pomu/util/git.py index 4c3e2ed..611b27c 100644 --- a/pomu/util/git.py +++ b/pomu/util/git.py @@ -1,23 +1,25 @@ """Miscellaneous utility functions for git structures""" from base64 import b16encode +import zlib from pomu.util.result import Result -def parse_tree(blob): +def parse_tree(blob, path=''): """Parses a git tree""" res = [] - leng, _, tree = blob.partition('\0') + leng, _, tree = blob.partition(b'\0') if not tree: return Result.Err('Invalid tree') - while len(data) > 0: - data = data[7:] # strip access code - name, _, data = data.partition('\0') - sha = b16encode(data[0:20]).decode('utf-8') - data = data[20:] + while len(tree) > 0: + mode, _, tree = tree.partition(b' ') + name, _, tree = tree.partition(b'\0') + sha = b16encode(tree[0:20]).decode('utf-8').lower() + tree = tree[20:] if not name or not sha: return Result.Err() - res.append((sha, name)) + is_dir = mode[0:1] != b'1' + res.append((is_dir, sha, path.encode('utf-8') + b'/' + name)) return Result.Ok(res) def parse_blob(blob): @@ -39,8 +41,8 @@ def commit_head(blob): def parse_object(obj): """Parses a git object""" data = zlib.decompress(obj) - if data[0:5] == 'blob ': + if data[0:5] == b'blob ': return parse_blob(data[5:]) - elif data[0:5] == 'tree ': + elif data[0:5] == b'tree ': return parse_tree(data[5:]) return Result.Err('Unsupported object type') diff --git a/pomu/util/remote.py b/pomu/util/remote.py new file mode 100644 index 0000000..20691e2 --- /dev/null +++ b/pomu/util/remote.py @@ -0,0 +1,38 @@ +""" +Utilities for remotes +""" + +from portage.versions import best + +from pomu.util.pkg import ver_str, cpv_split +from pomu.util.portage import misc_dirs + +def filelist_to_cpvs(tree): + """Converts a list of files to list of cpvs""" + res = [] + for opath in tree: + comps = opath.split('/') + if (opath.endswith('/') or + any(opath.startswith('/' + x + '/') for x in misc_dirs) or + len(comps) != 3 or + not comps[2].endswith('.ebuild')): + continue + category, name, ebuild = comps[0], comps[1], comps[2][:-7] + c, n, v, s, r = cpv_split(ebuild) + if category or n != name: + continue + ver = ver_str(v, s, r) + res.append((category, name, ver)) + return res + +def get_full_cpv(cpvs, name, category=None, version=None): + cpvl = cpvs.filter(lambda x: x[1] == name and (not category or x[0] == category)) + if not cpvl: return Result.Err() + if version: + cpvl = cpvl.filter(lambda x: x[2] == version)[:1] + b = best('{}/{}-{}'.format(c, n, v) for c, n, v in cpvl) + if b: + cat, name, v, s, r = cpv_split(b) + ver = ver_str(v, s, r) + return Result.Ok((cat, name, ver)) + return Result.Err() |