import logging from datetime import datetime from sqlobject import * from turbogears.database import PackageHub from turbogears import identity, config, flash from transifex.util import get_user_email log = logging.getLogger(__name__) hub = PackageHub('transifex') __connection__ = hub class Module(SQLObject): """ A module in an VCS which could be checked-out, committed, etc. """ name = UnicodeCol(alternateID=True, length=100, notNone=True) description = UnicodeCol(default=None) summary = UnicodeCol(default=None) repository = ForeignKey('Repository', default=None) scmmodule = UnicodeCol() branches = RelatedJoin('Branch') # only show this directory to users directory = UnicodeCol(default=None) filefilter = UnicodeCol(default=None) webpage = UnicodeCol(default=None) webfrontend = UnicodeCol(default=None) changelog = UnicodeCol(default=None) # a page with direct links to PO (eg. statistics) popage = UnicodeCol(default=None) logentries = MultipleJoin('ActionLog') lastUpdated = DateTimeCol(default=None) created = DateTimeCol(default=datetime.now) disabled = BoolCol(default=False) def __str__(self): return self.name def getBranches(self): return [b.name for b in self.branches] def refreshTime(self): """ Update the last-updated field. Called with each checkout() """ from datetime import datetime self.set(lastUpdated=datetime.now()) hub.commit() def canonicalPath(self, branchname=''): """ Returns the path of the stored module in three different useful forms: root (scratchdir), repository (all repo modules), module (whole module) and directory (the project itself). """ import os from turbogears import config if branchname == '': if self.repository.type == "cvs": branchname = 'HEAD' elif self.repository.type == "svn": branchname = 'trunk' elif self.repository.type == "hg": branchname = 'tip' elif self.repository.type == "git": branchname = 'master' ret = { 'root': config.get("scratchdir"), 'repository': os.path.join(config.get("scratchdir"), self.repository.type), 'module': os.path.join(config.get("scratchdir"), self.repository.type, self.scmmodule + "." + branchname),} if self.directory: ret.update({ 'directory': os.path.join(config.get("scratchdir"), self.repository.type, self.scmmodule + "." + branchname, self.directory)}) else: ret.update({ 'directory': os.path.join(config.get("scratchdir"), self.repository.type, self.scmmodule + "." + branchname)}) return ret def cleanLocalModule(self): """ Delete module directory in order to checkout instead of update """ #TODO: Make this branch-sensitive import os import sys modulePath = self.canonicalPath()['module'] if os.access(modulePath, os.W_OK): for root, dirs, files in os.walk(modulePath, topdown=False): for name in files: os.remove(os.path.join(root, name)) for name in dirs: os.rmdir(os.path.join(root, name)) os.rmdir(modulePath) return 1 else: return 0 def checkout(self, branchname=None, target=None, addRepoTypePrefix=True): """ Check-out a module's branch in `target/modulename.branchname/`. If addRepoTypePrefix is True, at `target/repotype/modulename.branchname`. If clean is True, delete existing directory and always checkout (no update). """ import os from transifex.util import run_commands if branchname == None: if self.repository.type == "cvs": branchname = 'HEAD' elif self.repository.type == "svn": branchname = 'trunk' elif self.repository.type == "hg": branchname = 'tip' elif self.repository.type == "git": branchname = 'master' else: log.error("Woa. We shouldn't be here.") return log.debug("Checkout called with no branch name. Assuming " "'%(branch)s'." % {'branch': branchname}) try: branch = Branch.selectBy(name=branchname)[0] except SQLObjectNotFound: return 0 if not branch in self.branches: log.error("ERROR. Module %(module)s does not have a branch " "registered with the name " "'%(branch)s'." % {'module' : self.scmmodule, 'branch' : branchname}) return 0 #FIXME: this doesn't work, need to find out why. if not target: target = self.canonicalPath()['root'] # define basic directories moduledir = self.scmmodule + "." + branch.name if addRepoTypePrefix: target = os.path.join(target, self.repository.type) modulepath = os.path.join(target, moduledir) # the target (parent) dir must exist # TODO: add checks here try: os.makedirs(target) except: pass coms = [] if os.access(modulepath, os.X_OK | os.W_OK): # FIXME: Make sure all updates are clean (overwrite locally # changed files) # Check if this might cause any problems with simultaneous commits? log.debug("Updating '%(mod)s', branch %(branch)s" " (%(type)s)" % {'mod': self.scmmodule, 'branch': branch.name, 'type': self.repository.type}) if self.repository.type == "cvs": coms.append({'args': ["cvs", "-z4", "up", "-PdC"], 'cwd': modulepath}) elif self.repository.type == "svn": coms.append({'args': ["svn", "revert", "-R", "."], 'cwd': modulepath}) coms.append({'args': ["svn", "update"], 'cwd': modulepath}) elif self.repository.type == "hg": coms.append({'args': ["hg", "revert", "--all"], 'cwd': modulepath}) coms.append({'args': ["hg", "pull", "-u"], 'cwd': modulepath}) coms.append({'args': ["hg", "update", branch.name], 'cwd': modulepath}) elif self.repository.type == "git": coms.append({'args': ["git", "reset", "--hard", "origin"], 'cwd': modulepath}) coms.append({'args': ["git", "clean", "-d"], 'cwd': modulepath}) coms.append({'args': ["git", "fetch"], 'cwd': modulepath}) # coms.append({'args': ["git", "checkout", branch.name], # 'cwd': modulepath}) else: log.debug("Checking out '%(mod)s', branch %(branch)s" " (%(type)s)" % {'dir': target, 'mod': self.scmmodule, 'branch': branch.name, 'type': self.repository.type}) if self.repository.type == "cvs": # CVS HEAD should be checked out without a branch tag; # otherwise commit doesn't work if (branch.name == 'HEAD'): coms.append({'args': ["cvs", "-d", self.repository.root, "-z4", "co", "-d", moduledir, self.scmmodule], 'cwd': target}) else: coms.append({'args': ["cvs", "-d", self.repository.root, "-z4", "co", "-d", moduledir, branch.name, self.scmmodule], 'cwd': target}) elif self.repository.type == 'svn': if branch.name == 'trunk': scmpath = self.repository.root + "/" + self.scmmodule + \ '/' + branch.name else: scmpath = self.repository.root + "/" + self.scmmodule + \ '/branches/' + branch.name coms.append({'args': ["svn", "--non-interactive", "co", scmpath, moduledir], 'cwd': target}) elif self.repository.type == "hg": scmpath = self.repository.root + "/" + self.scmmodule coms.append({'args': ["hg", "clone", scmpath, moduledir], 'cwd': target}) coms.append({'args': ["hg", "update", branch.name], 'cwd': modulepath}) elif self.repository.type == "git": scmpath = self.repository.root + "/" + self.scmmodule coms.append({'args': ["git", "clone", scmpath, moduledir], 'cwd': target}) coms.append({'args': ["git", "checkout", branch.name], 'cwd': modulepath}) # run commands run_ret = run_commands(coms) if not run_ret: log.debug("Clone/checkout/update succeeded.") self.refreshTime() return 1 else: log.error("Clone/checkout/update failed for module %(module)s" ", branch '%(branch)s'." % {'module' : self.scmmodule, 'branch' : branchname}) def commit(self, branch, commit_files, user, commit_message=''): """ Commit files to an already checked-out module/branch. commit_files is a list of (filename, file_contents) of files to be committed Their paths should be relative to the module.directory. """ import os import codecs import subprocess from time import (gmtime, strftime) from transifex.util import run_commands, get_user_info log.debug("Commit procedure initiated for user %s, files: %s" % ( user, ', '.join([f[0] for f in commit_files])) ) modulePath = self.canonicalPath(branchname=branch.name)['directory'] for (filename, file_contents) in commit_files: newFile = False filePath = os.path.join(modulePath, filename) if os.path.exists(filePath) and not os.access(filePath, os.W_OK | os.R_OK): log.error("File exists, no permissions to" " apply patch (%s)" % (filePath)) return 0 if not os.path.exists(filePath): if not os.access(os.path.dirname(filePath), os.W_OK): log.error("File doesn't exists, no write permissions" " in dir %s" % (os.path.dirname(filePath))) return 0 else: log.debug("New file requested.") newFile = True if not os.path.normpath(filePath).startswith(modulePath): log.error("Something fishy is happening here with requested" " paths (%s)." % filePath) flash(_("Something fishy is happening here...")) return 0 log.debug("Saving file %s" % (filePath)) # Handling the file as binary, to avoid any encoding issues # We're just overwriting it with whatever file_contents has try: f = open(os.path.join(modulePath, filePath), "wb") f.write(file_contents) f.close() except IOError: log.error("Can't save uploaded file.") return coms=[] # Do the commit # TODO: Some maintainers might not want to push to main repo. # FIXME: Only commit files that have been changed. msg_esc = commit_message.replace("'", "\'") if self.repository.type == "cvs": if newFile: coms.append({'args': ["cvs", "add", filename], 'cwd': modulePath}) coms.append({'args': ["cvs", "-z4", "commit", "-m", msg_esc], 'cwd': modulePath}) elif self.repository.type == "svn": if newFile: coms.append({'args': ["svn", "add", filename], 'cwd': modulePath}) coms.append({'args': ["svn", "--non-interactive", "commit", "-m", msg_esc, "."], 'cwd': modulePath}) elif self.repository.type == "hg": coms.append({'args': ["hg", "commit", "-A", "-m", msg_esc, "-u", get_user_info(), "."], 'cwd': modulePath}) coms.append({'args': ["hg", "push"], 'cwd': modulePath}) elif self.repository.type == "git": coms.append({'args': ["git", "config", "user.name", identity.current.user.display_name], 'cwd': modulePath}) coms.append({'args': ["git", "config", "user.email", get_user_email(identity)], 'cwd': modulePath}) if newFile: coms.append({'args': ["git", "add", filename], 'cwd': modulePath}) coms.append({'args': ["git", "commit", "-m", msg_esc, "."], 'cwd': modulePath}) coms.append({'args': ["git", "pull"], 'cwd': modulePath}) coms.append({'args': ["git", "push", "origin", branch.name], 'cwd': modulePath}) # run commit commands run_ret = run_commands(coms) if not run_ret: log.debug("Commit/push succeeded.") return 1 else: log.error("Commit/push failed for module %(module)s" ", branch '%(branch)s'." % {'module' : self.scmmodule, 'branch' : branch.name}) return 0 def getFiles(self, branch, ignoreWildcard=None): import os, re from turbogears import config if not ignoreWildcard: ignoreWildcard = config.get("ignore_wildcard_list") modulePath = self.canonicalPath(branchname=branch.name)['directory'] moduleFiles = [] if os.access(modulePath, os.X_OK | os.R_OK): for root, dirs, files in os.walk(modulePath, topdown=True): for name in files: filePath = os.path.join(root, name) # ignore filters if re.compile(ignoreWildcard).match(filePath): continue # allow filter if self.filefilter: if not re.compile(self.filefilter).match(filePath): continue #TODO: I don't like this check too much if self.changelog and filePath.endswith(self.changelog): log.debug("Skipping changelog file %s" % filePath) continue moduleFiles.append(filePath) # trim path from file names filesShown = [k[len(modulePath)+1:] for k in moduleFiles if k.startswith(modulePath)] filesShown.sort() return filesShown class Repository(SQLObject): """ A VCS repository. """ name = UnicodeCol(alternateID=True, length=100, notNone=True) description = UnicodeCol(default=None) summary = UnicodeCol(default=None) type = EnumCol(enumValues=['cvs', 'svn', 'hg', 'git', 'bzr'], default='cvs') root = UnicodeCol(default=None) authtype = EnumCol(enumValues=['none', 'rsa', 'dsa', 'password'], default='none') password = UnicodeCol(default=None) ssh_key = UnicodeCol(default=None) webpage = UnicodeCol(default=None) webfrontend = UnicodeCol(default=None) modules = MultipleJoin('Module') created = DateTimeCol(default=datetime.now) disabled = BoolCol(default=False) def __str__(self): return self.name class Branch(SQLObject): """ A VCS branch. """ name = UnicodeCol(alternateID=True, length=100, notNone=True) modules = RelatedJoin('Module') description = UnicodeCol(default=None) stringfrozen = BoolCol(default=False) def __str__(self): return self.name class ActionLog(SQLObject): """ Log for submits of a module """ user = ForeignKey('User', default=None) userinfo = UnicodeCol(default=None) module = ForeignKey('Module', default=None) branch = ForeignKey('Branch', default=None) filenames = UnicodeCol(default=None) type = EnumCol(enumValues=['submit', 'addmodule', 'addrepo', 'preview'], default='submit') summary = UnicodeCol(default=None) changelog = UnicodeCol(default=None) ip = StringCol(length=40, default=None) diff = BLOBCol(default=None) timestamp = DateTimeCol(default=datetime.now) def __str__(self): return self.summary # identity models. class Visit(SQLObject): """ A visit to your site """ class sqlmeta: table = 'visit' visit_key = StringCol(length=40, alternateID=True, alternateMethodName='by_visit_key') created = DateTimeCol(default=datetime.now) expiry = DateTimeCol() def lookup_visit(cls, visit_key): try: return cls.by_visit_key(visit_key) except SQLObjectNotFound: return None lookup_visit = classmethod(lookup_visit) class VisitIdentity(SQLObject): """ A Visit that is link to a User object """ visit_key = StringCol(length=40, alternateID=True, alternateMethodName='by_visit_key') user_id = IntCol() class Group(SQLObject): """ An ultra-simple group definition. """ # names like "Group", "Order" and "User" are reserved words in SQL # so we set the name to something safe for SQL class sqlmeta: table = 'tg_group' group_name = UnicodeCol(length=16, alternateID=True, alternateMethodName='by_group_name') display_name = UnicodeCol() created = DateTimeCol(default=datetime.now) # collection of all users belonging to this group users = RelatedJoin('User', intermediateTable='user_group', joinColumn='group_id', otherColumn='user_id') # collection of all permissions for this group permissions = RelatedJoin('Permission', joinColumn='group_id', intermediateTable='group_permission', otherColumn='permission_id') class User(SQLObject): """ Reasonably basic User definition. Probably would want additional attributes. """ # names like "Group", "Order" and "User" are reserved words in SQL # so we set the name to something safe for SQL class sqlmeta: table = 'tg_user' user_name = UnicodeCol(length=16, alternateID=True, alternateMethodName='by_user_name') email_address = UnicodeCol(length=100, alternateID=True, alternateMethodName='by_email_address') display_name = UnicodeCol() password = UnicodeCol(length=40) created = DateTimeCol(default=datetime.now) # groups this user belongs to groups = RelatedJoin('Group', intermediateTable='user_group', joinColumn='user_id', otherColumn='group_id') def _get_permissions(self): perms = set() for g in self.groups: perms = perms | set(g.permissions) return perms def _set_password(self, cleartext_password): "Runs cleartext_password through the hash algorithm before saving." password_hash = identity.encrypt_password(cleartext_password) self._SO_set_password(password_hash) def set_password_raw(self, password): "Saves the password as-is to the database." self._SO_set_password(password) class Permission(SQLObject): """ A relationship that determines what each Group can do """ permission_name = UnicodeCol(length=16, alternateID=True, alternateMethodName='by_permission_name') description = UnicodeCol() groups = RelatedJoin('Group', intermediateTable='group_permission', joinColumn='permission_id', otherColumn='group_id')