| """ | |
| BitBake 'Fetch' implementation for perforce | |
| Supported SRC_URI options are: | |
| - module | |
| The top-level location to fetch while preserving the remote paths | |
| The value of module can point to either a directory or a file. The result, | |
| in both cases, is that the fetcher will preserve all file paths starting | |
| from the module path. That is, the top-level directory in the module value | |
| will also be the top-level directory in P4DIR. | |
| - remotepath | |
| If the value "keep" is given, the full depot location of each file is | |
| preserved in P4DIR. This option overrides the effect of the module option. | |
| """ | |
| # Copyright (C) 2003, 2004 Chris Larson | |
| # Copyright (C) 2016 Kodak Alaris, Inc. | |
| # | |
| # SPDX-License-Identifier: GPL-2.0-only | |
| # | |
| # Based on functions from the base bb module, Copyright 2003 Holger Schurig | |
| import os | |
| import bb | |
| import shlex | |
| from bb.fetch import FetchMethod | |
| from bb.fetch import FetchError | |
| from bb.fetch import logger | |
| from bb.fetch import runfetchcmd | |
| class PerforceProgressHandler (bb.progress.BasicProgressHandler): | |
| """ | |
| Implements basic progress information for perforce, based on the number of | |
| files to be downloaded. | |
| The p4 print command will print one line per file, therefore it can be used | |
| to "count" the number of files already completed and give an indication of | |
| the progress. | |
| """ | |
| def __init__(self, d, num_files): | |
| self._num_files = num_files | |
| self._count = 0 | |
| super(PerforceProgressHandler, self).__init__(d) | |
| # Send an initial progress event so the bar gets shown | |
| self._fire_progress(-1) | |
| def write(self, string): | |
| self._count = self._count + 1 | |
| percent = int(100.0 * float(self._count) / float(self._num_files)) | |
| # In case something goes wrong, we try to preserve our sanity | |
| if percent > 100: | |
| percent = 100 | |
| self.update(percent) | |
| super(PerforceProgressHandler, self).write(string) | |
| class Perforce(FetchMethod): | |
| """ Class to fetch from perforce repositories """ | |
| def supports(self, ud, d): | |
| """ Check to see if a given url can be fetched with perforce. """ | |
| return ud.type in ['p4'] | |
| def urldata_init(self, ud, d): | |
| """ | |
| Initialize perforce specific variables within url data. If P4CONFIG is | |
| provided by the env, use it. If P4PORT is specified by the recipe, use | |
| its values, which may override the settings in P4CONFIG. | |
| """ | |
| ud.basecmd = shlex.split(d.getVar("FETCHCMD_p4") or "") or ["p4"] | |
| ud.dldir = d.getVar("P4DIR") or (d.getVar("DL_DIR") + "/p4") | |
| path = ud.url.split('://')[1] | |
| path = path.split(';')[0] | |
| delim = path.find('@'); | |
| if delim != -1: | |
| (ud.user, ud.pswd) = path.split('@')[0].split(':') | |
| ud.path = path.split('@')[1] | |
| else: | |
| ud.path = path | |
| ud.usingp4config = False | |
| p4port = d.getVar('P4PORT') | |
| if p4port: | |
| logger.debug('Using recipe provided P4PORT: %s' % p4port) | |
| ud.host = p4port | |
| else: | |
| logger.debug('Trying to use P4CONFIG to automatically set P4PORT...') | |
| ud.usingp4config = True | |
| p4cmd = ud.basecmd + ['info'] | |
| bb.fetch.check_network_access(d, p4cmd, ud.url) | |
| output = runfetchcmd(p4cmd, d, True) | |
| ud.host = None | |
| for line in output.splitlines(): | |
| if "Server address" in line: | |
| ud.host = line.split(': ')[1].strip() | |
| break | |
| logger.debug('Determined P4PORT to be: %s' % ud.host) | |
| if not ud.host: | |
| raise FetchError('Could not determine P4PORT from P4CONFIG') | |
| # Fetcher options | |
| ud.module = ud.parm.get('module') | |
| ud.keepremotepath = (ud.parm.get('remotepath', '') == 'keep') | |
| if ud.path.find('/...') >= 0: | |
| ud.pathisdir = True | |
| else: | |
| ud.pathisdir = False | |
| # Avoid using the "/..." syntax in SRC_URI when a module value is given | |
| if ud.pathisdir and ud.module: | |
| raise FetchError('SRC_URI depot path cannot not end in /... when a module value is given') | |
| cleanedpath = ud.path.replace('/...', '').replace('/', '.') | |
| cleanedhost = ud.host.replace(':', '.') | |
| cleanedmodule = "" | |
| # Merge the path and module into the final depot location | |
| if ud.module: | |
| if ud.module.find('/') == 0: | |
| raise FetchError('module cannot begin with /') | |
| ud.path = os.path.join(ud.path, ud.module) | |
| # Append the module path to the local pkg name | |
| cleanedmodule = ud.module.replace('/...', '').replace('/', '.') | |
| cleanedpath += '--%s' % cleanedmodule | |
| ud.pkgdir = os.path.join(ud.dldir, cleanedhost, cleanedpath) | |
| ud.setup_revisions(d) | |
| ud.localfile = d.expand('%s_%s_%s_%s.tar.gz' % (cleanedhost, cleanedpath, cleanedmodule, ud.revision)) | |
| def _buildp4command(self, ud, d, command, depot_filename=None): | |
| """ | |
| Build a p4 commandline. Valid commands are "changes", "print", and | |
| "files". depot_filename is the full path to the file in the depot | |
| including the trailing '#rev' value. | |
| """ | |
| p4opt = [] | |
| if ud.user: | |
| p4opt += ['-u', ud.user] | |
| if ud.pswd: | |
| p4opt += ['-P', ud.pswd] | |
| if ud.host and not ud.usingp4config: | |
| p4opt += ['-p', ud.host] | |
| if hasattr(ud, 'revision') and ud.revision: | |
| pathnrev = '%s@%s' % (ud.path, ud.revision) | |
| else: | |
| pathnrev = '%s' % (ud.path) | |
| if depot_filename: | |
| if ud.keepremotepath: | |
| # preserve everything, remove the leading // | |
| filename = depot_filename.lstrip('/') | |
| elif ud.module: | |
| # remove everything up to the module path | |
| modulepath = ud.module.rstrip('/...') | |
| filename = depot_filename[depot_filename.rfind(modulepath):] | |
| elif ud.pathisdir: | |
| # Remove leading (visible) path to obtain the filepath | |
| filename = depot_filename[len(ud.path)-1:] | |
| else: | |
| # Remove everything, except the filename | |
| filename = depot_filename[depot_filename.rfind('/'):] | |
| filename = filename[:filename.find('#')] # Remove trailing '#rev' | |
| if command == 'changes': | |
| p4cmd = ud.basecmd + p4opt + ['changes', '-m', '1', '//%s' % pathnrev] | |
| elif command == 'print': | |
| if depot_filename is not None: | |
| p4cmd = ud.basecmd + p4opt + ['print', '-o', 'p4/%s' % filename, depot_filename] | |
| else: | |
| raise FetchError('No depot file name provided to p4 %s' % command, ud.url) | |
| elif command == 'files': | |
| p4cmd = ud.basecmd + p4opt + ['files', '//%s' % pathnrev] | |
| else: | |
| raise FetchError('Invalid p4 command %s' % command, ud.url) | |
| return p4cmd | |
| def _p4listfiles(self, ud, d): | |
| """ | |
| Return a list of the file names which are present in the depot using the | |
| 'p4 files' command, including trailing '#rev' file revision indicator | |
| """ | |
| p4cmd = self._buildp4command(ud, d, 'files') | |
| bb.fetch.check_network_access(d, p4cmd, ud.url) | |
| p4fileslist = runfetchcmd(p4cmd, d, True) | |
| p4fileslist = [f.rstrip() for f in p4fileslist.splitlines()] | |
| if not p4fileslist: | |
| raise FetchError('Unable to fetch listing of p4 files from %s@%s' % (ud.host, ud.path)) | |
| count = 0 | |
| filelist = [] | |
| for filename in p4fileslist: | |
| item = filename.split(' - ') | |
| lastaction = item[1].split() | |
| logger.debug('File: %s Last Action: %s' % (item[0], lastaction[0])) | |
| if lastaction[0] == 'delete': | |
| continue | |
| filelist.append(item[0]) | |
| return filelist | |
| def download(self, ud, d): | |
| """ Get the list of files, fetch each one """ | |
| filelist = self._p4listfiles(ud, d) | |
| if not filelist: | |
| raise FetchError('No files found in depot %s@%s' % (ud.host, ud.path)) | |
| bb.utils.remove(ud.pkgdir, True) | |
| bb.utils.mkdirhier(ud.pkgdir) | |
| progresshandler = PerforceProgressHandler(d, len(filelist)) | |
| for afile in filelist: | |
| p4fetchcmd = self._buildp4command(ud, d, 'print', afile) | |
| bb.fetch.check_network_access(d, p4fetchcmd, ud.url) | |
| runfetchcmd(p4fetchcmd, d, workdir=ud.pkgdir, log=progresshandler) | |
| runfetchcmd(['tar', '-czf', ud.localpath, 'p4'], d, cleanup=[ud.localpath], workdir=ud.pkgdir) | |
| def clean(self, ud, d): | |
| """ Cleanup p4 specific files and dirs""" | |
| bb.utils.remove(ud.localpath) | |
| bb.utils.remove(ud.pkgdir, True) | |
| def supports_srcrev(self): | |
| return True | |
| def _revision_key(self, ud, d, name): | |
| """ Return a unique key for the url """ | |
| return 'p4:%s' % ud.pkgdir | |
| def _latest_revision(self, ud, d, name): | |
| """ Return the latest upstream scm revision number """ | |
| p4cmd = self._buildp4command(ud, d, "changes") | |
| bb.fetch.check_network_access(d, p4cmd, ud.url) | |
| tip = runfetchcmd(p4cmd, d, True) | |
| if not tip: | |
| raise FetchError('Could not determine the latest perforce changelist') | |
| tipcset = tip.split(' ')[1] | |
| logger.debug('p4 tip found to be changelist %s' % tipcset) | |
| return tipcset | |
| def sortable_revision(self, ud, d, name): | |
| """ Return a sortable revision number """ | |
| return False, self._build_revision(ud, d) | |
| def _build_revision(self, ud, d): | |
| return ud.revision | |