from datetime import datetime import logging from turbogears.database import metadata, mapper, get_engine # import some basic SQLAlchemy classes for declaring the data model # (see http://www.sqlalchemy.org/docs/04/ormtutorial.html) from sqlalchemy import Table, Column, ForeignKey from sqlalchemy.orm import relation # import some datatypes for table columns from SQLAlchemy # (see http://www.sqlalchemy.org/docs/04/types.html for more) from sqlalchemy import PickleType, String, Boolean, Unicode, UnicodeText, Integer, DateTime from turbogears import identity, config, flash from transifex.util import get_user_email log = logging.getLogger(__name__) # Bind us to the database defined in the config file. #get_engine() action_log_table = Table('action_log', metadata, Column(u'id', Integer(), primary_key=True, nullable=False), Column(u'user_id', Integer, ForeignKey('tg_user.user_id'), index=True), Column(u'userinfo', Unicode(255)), Column(u'module_id', Integer, ForeignKey('module.id'), index=True), Column(u'branch_id', Integer, ForeignKey('branch.id'), index=True), Column(u'filenames', UnicodeText()), Column(u'type', Unicode(32), default=u'submit', nullable=False), Column(u'summary', Unicode(255)), Column(u'changelog', UnicodeText()), Column(u'ip', String(length=40)), Column(u'diff', UnicodeText()), Column(u'timestamp', DateTime(timezone=False), default=datetime.now), ) branch_table = Table('branch', metadata, Column(u'id', Integer(), primary_key=True, nullable=False), Column(u'name', Unicode(100), unique=True, nullable=False), Column(u'description', Unicode(255)), Column(u'stringfrozen', Boolean(), default=False), ) branch_module_table = Table('branch_module', metadata, Column('branch_id', Integer, ForeignKey('branch.id', onupdate='CASCADE', ondelete='CASCADE')), Column('module_id', Integer, ForeignKey('module.id', onupdate='CASCADE', ondelete='CASCADE')) ) module_table = Table('module', metadata, Column(u'id', Integer(), primary_key=True, nullable=False), Column(u'name', Unicode(100), unique=True, nullable=False), Column(u'description', Unicode(255)), Column(u'summary', Unicode(255)), Column(u'repository_id', Integer, ForeignKey('repository.id'), index=True), Column(u'scmmodule', Unicode(255)), Column(u'directory', Unicode(255)), Column(u'filefilter', Unicode(255)), Column(u'webpage', Unicode(255)), Column(u'webfrontend', Unicode(255)), Column(u'changelog', Unicode(255)), Column(u'popage', Unicode(255)), Column(u'last_updated', DateTime(timezone=False), default=datetime.now), Column(u'created', DateTime(timezone=False), default=datetime.now), Column(u'disabled', Boolean(), default=False), ) repository_table = Table('repository', metadata, Column(u'id', Integer(), primary_key=True, nullable=False), Column(u'name', Unicode(100), unique=True, nullable=False), Column(u'description', Unicode(255)), Column(u'summary', Unicode(255)), Column(u'type', Unicode(32), default=u'cvs', nullable=False), Column(u'root', Unicode(255)), Column(u'authtype', Unicode(32), default=u'none',nullable=False), Column(u'password', Unicode(255)), Column(u'ssh_key', Unicode(255)), Column(u'webpage', Unicode(255)), Column(u'webfrontend', Unicode(255)), Column(u'created', DateTime(timezone=False), default=datetime.now), Column(u'disabled', Boolean(), default=False), ) # Identity visits_table = Table('visit', metadata, Column('visit_key', String(40), primary_key=True), Column('created', DateTime, nullable=False, default=datetime.now), Column('expiry', DateTime) ) visit_identity_table = Table('visit_identity', metadata, Column('visit_key', String(40), primary_key=True), Column('user_id', Integer, ForeignKey('tg_user.user_id'), index=True) ) groups_table = Table('tg_group', metadata, Column('group_id', Integer, primary_key=True), Column('group_name', Unicode(16), unique=True), Column('display_name', Unicode(255)), Column('created', DateTime, default=datetime.now) ) users_table = Table('tg_user', metadata, Column('user_id', Integer, primary_key=True), Column('user_name', Unicode(16), unique=True), Column('email_address', Unicode(255), unique=True), Column('display_name', Unicode(255)), Column('password', Unicode(40)), Column('created', DateTime, default=datetime.now) ) permissions_table = Table('permission', metadata, Column('permission_id', Integer, primary_key=True), Column('permission_name', Unicode(16), unique=True), Column('description', Unicode(255)) ) user_group_table = Table('user_group', metadata, Column('user_id', Integer, ForeignKey('tg_user.user_id', onupdate='CASCADE', ondelete='CASCADE')), Column('group_id', Integer, ForeignKey('tg_group.group_id', onupdate='CASCADE', ondelete='CASCADE')) ) group_permission_table = Table('group_permission', metadata, Column('group_id', Integer, ForeignKey('tg_group.group_id', onupdate='CASCADE', ondelete='CASCADE')), Column('permission_id', Integer, ForeignKey('permission.permission_id', onupdate='CASCADE', ondelete='CASCADE')) ) # Database Session session_table = Table('session', metadata, Column('id', String(40), primary_key=True), Column('data', PickleType), Column('expiration_time', DateTime) ) # the identity model class Visit(object): """ A visit to your site """ def lookup_visit(cls, visit_key): return cls.query.get(visit_key) lookup_visit = classmethod(lookup_visit) class VisitIdentity(object): """ A Visit that is link to a User object """ pass class Group(object): """ An ultra-simple group definition. """ pass class User(object): """ Reasonably basic User definition. Probably would want additional attributes. """ def permissions(self): perms = set() for g in self.groups: perms |= set(g.permissions) return perms permissions = property(permissions) @classmethod def by_email_address(cls, email): """ A class method that can be used to search users based on their email addresses since it is unique. """ return cls.query.filter_by(email_address=email).first() @classmethod def by_user_name(cls, username): """ A class method that permits to search users based on their user_name attribute. """ return cls.query.filter_by(user_name=username).first() def _set_password(self, password): """ encrypts password on the fly using the encryption algo defined in the configuration """ self._password = identity.encrypt_password(password) def _get_password(self): """ returns password """ return self._password password = property(_get_password, _set_password) class Permission(object): """ A relationship that determines what each Group can do """ pass # Session Model class Session(object): """ Session """ class Module(object): """ A module in an VCS which could be checked-out, committed, etc. """ def __str__(self): return self.name @classmethod def byName(cls, name): return cls.query.filter_by(name=name).one() def getBranches(self): return [b.name for b in self.branches] def refreshTime(self): """Update the last-updated field. Called with each checkout().""" from datetime import datetime self.last_updated=datetime.now() def canonicalPath(self, branchname=''): """ Returns the path of the stored module in three different useful forms: root (scratchdir), repository (all repo modules), module (whole module) and directory (the project itself). """ import os from turbogears import config if branchname == '': if self.repository.type == "cvs": branchname = 'HEAD' elif self.repository.type == "svn": branchname = 'trunk' elif self.repository.type == "hg": branchname = 'tip' elif self.repository.type == "git": branchname = 'master' ret = { 'root': config.get("scratchdir"), 'repository': os.path.join(config.get("scratchdir"), self.repository.type), 'module': os.path.join(config.get("scratchdir"), self.repository.type, "%s.%s" % (self.scmmodule, branchname)),} if self.directory: ret.update({ 'directory': os.path.join(config.get("scratchdir"), self.repository.type, "%s.%s" % (self.scmmodule, branchname), self.directory)}) else: ret.update({ 'directory': os.path.join(config.get("scratchdir"), self.repository.type, "%s.%s" % (self.scmmodule, branchname))}) return ret def cleanLocalModule(self): """ Delete module directory in order to checkout instead of update """ #TODO: Make this branch-sensitive import os import sys modulePath = self.canonicalPath()['module'] if os.access(modulePath, os.W_OK): for root, dirs, files in os.walk(modulePath, topdown=False): for name in files: os.remove(os.path.join(root, name)) for name in dirs: os.rmdir(os.path.join(root, name)) os.rmdir(modulePath) return 1 else: return 0 def checkout(self, branchname=None, target=None, addRepoTypePrefix=True): """ Check-out a module's branch in `target/modulename.branchname/`. If addRepoTypePrefix is True, at `target/repotype/modulename.branchname`. If clean is True, delete existing directory and always checkout (no update). """ import os from transifex.util import run_commands if branchname == None: if self.repository.type == "cvs": branchname = 'HEAD' elif self.repository.type == "svn": branchname = 'trunk' elif self.repository.type == "hg": branchname = 'tip' elif self.repository.type == "git": branchname = 'master' else: log.error("Woa. We shouldn't be here.") return log.debug("Checkout called with no branch name. Assuming " "'%(branch)s'." % {'branch': branchname}) try: branch = Branch.byName(branchname) except Exception, e: # TODO better exception handling return 0 if not branch in self.branches: log.error("ERROR. Module %(module)s does not have a branch " "registered with the name " "'%(branch)s'." % {'module' : self.scmmodule, 'branch' : branchname}) return 0 #FIXME: this doesn't work, need to find out why. if not target: target = self.canonicalPath()['root'] # define basic directories moduledir = self.scmmodule + "." + branch.name if addRepoTypePrefix: target = os.path.join(target, self.repository.type) modulepath = os.path.join(target, moduledir) # the target (parent) dir must exist # TODO: add checks here try: os.makedirs(target) except Exception ,e: log.debug(e) coms = [] if os.access(modulepath, os.X_OK | os.W_OK): # FIXME: Make sure all updates are clean (overwrite locally # changed files) # Check if this might cause any problems with simultaneous commits? log.debug("Updating '%(mod)s', branch %(branch)s" " (%(type)s)" % {'mod': self.scmmodule, 'branch': branch.name, 'type': self.repository.type}) if self.repository.type == "cvs": coms.append({'args': ["cvs", "-z4", "up", "-PdC"], 'cwd': modulepath}) elif self.repository.type == "svn": coms.append({'args': ["svn", "revert", "-R", "."], 'cwd': modulepath}) coms.append({'args': ["svn", "update"], 'cwd': modulepath}) elif self.repository.type == "hg": coms.append({'args': ["hg", "revert", "--all"], 'cwd': modulepath}) coms.append({'args': ["hg", "pull", "-u"], 'cwd': modulepath}) coms.append({'args': ["hg", "update", branch.name], 'cwd': modulepath}) elif self.repository.type == "git": coms.append({'args': ["git", "fetch", "origin"], 'cwd': modulepath}) coms.append({'args': ["git", "reset", "--hard", "origin/%s" % branch.name], 'cwd': modulepath}) else: log.debug("Checking out '%(mod)s', branch %(branch)s" " (%(type)s)" % {'dir': target, 'mod': self.scmmodule, 'branch': branch.name, 'type': self.repository.type}) if self.repository.type == "cvs": # CVS HEAD should be checked out without a branch tag; # otherwise commit doesn't work if (branch.name == 'HEAD'): coms.append({'args': ["cvs", "-d", self.repository.root, "-z4", "co", "-d", moduledir, self.scmmodule], 'cwd': target}) else: coms.append({'args': ["cvs", "-d", self.repository.root, "-z4", "co", "-d", moduledir, branch.name, self.scmmodule], 'cwd': target}) elif self.repository.type == 'svn': if branch.name == 'trunk': scmpath = self.repository.root + "/" + self.scmmodule + \ '/' + branch.name else: scmpath = self.repository.root + "/" + self.scmmodule + \ '/branches/' + branch.name coms.append({'args': ["svn", "--non-interactive", "co", scmpath, moduledir], 'cwd': target}) elif self.repository.type == "hg": scmpath = self.repository.root + "/" + self.scmmodule coms.append({'args': ["hg", "clone", scmpath, moduledir], 'cwd': target}) coms.append({'args': ["hg", "update", branch.name], 'cwd': modulepath}) elif self.repository.type == "git": scmpath = self.repository.root + "/" + self.scmmodule coms.append({'args': ["git", "clone", scmpath, moduledir], 'cwd': target}) if branch != 'master': coms.append({'args': ["git", "branch", "%s" % branch.name, "origin/%s" % branch.name], 'cwd': modulepath}) coms.append({'args': ["git", "checkout", branch.name], 'cwd': modulepath}) # run commands run_ret = run_commands(coms) if not run_ret: log.debug("Clone/checkout/update succeeded.") self.refreshTime() return 1 else: log.error("Clone/checkout/update failed for module %(module)s" ", branch '%(branch)s'." % {'module' : self.scmmodule, 'branch' : branchname}) def commit(self, branch, commit_files, user, commit_message=''): """ Commit files to an already checked-out module/branch. commit_files is a list of (filename, file_contents) of files to be committed Their paths should be relative to the module.directory. """ import os import codecs import subprocess from time import (gmtime, strftime) from transifex.util import run_commands, get_user_info log.debug("Commit procedure initiated for user %s, files: %s" % ( user, ', '.join([f[0] for f in commit_files])) ) modulePath = self.canonicalPath(branchname=branch.name)['directory'] for (filename, file_contents) in commit_files: newFile = False filePath = os.path.join(modulePath, filename) if os.path.exists(filePath) and not os.access(filePath, os.W_OK | os.R_OK): log.error("File exists, no permissions to" " apply patch (%s)" % (filePath)) return 0 if not os.path.exists(filePath): if not os.access(os.path.dirname(filePath), os.W_OK): log.error("File doesn't exists, no write permissions" " in dir %s" % (os.path.dirname(filePath))) return 0 else: log.debug("New file requested.") newFile = True if not os.path.normpath(filePath).startswith(modulePath): log.error("Something fishy is happening here with requested" " paths (%s)." % filePath) flash(_("Something fishy is happening here...")) return 0 log.debug("Saving file %s" % (filePath)) # Handling the file as binary, to avoid any encoding issues # We're just overwriting it with whatever file_contents has try: f = open(os.path.join(modulePath, filePath), "wb") f.write(file_contents) f.close() except IOError: log.error("Can't save uploaded file.") return coms=[] # Do the commit # TODO: Some maintainers might not want to push to main repo. # FIXME: Only commit files that have been changed. msg_esc = commit_message.replace("'", "\'") if self.repository.type == "cvs": if newFile: coms.append({'args': ["cvs", "add", filename], 'cwd': modulePath}) coms.append({'args': ["cvs", "-z4", "commit", "-m", msg_esc], 'cwd': modulePath}) elif self.repository.type == "svn": if newFile: coms.append({'args': ["svn", "add", filename], 'cwd': modulePath}) coms.append({'args': ["svn", "--non-interactive", "commit", "-m", msg_esc, "."], 'cwd': modulePath}) elif self.repository.type == "hg": coms.append({'args': [ "hg", "commit", "-A", "-m", msg_esc, "-u", '%s <%s>' % (identity.current.user.display_name, get_user_email(identity)), "."], 'cwd': modulePath}) coms.append({'args': ["hg", "push"], 'cwd': modulePath}) elif self.repository.type == "git": coms.append({'args': ["git", "config", "user.name", identity.current.user.display_name], 'cwd': modulePath}) coms.append({'args': ["git", "config", "user.email", get_user_email(identity)], 'cwd': modulePath}) if newFile: coms.append({'args': ["git", "add", filename], 'cwd': modulePath}) coms.append({'args': ["git", "commit", "-m", msg_esc, "."], 'cwd': modulePath}) coms.append({'args': ["git", "push", "origin", "%s:%s" % (branch.name, branch.name) ], 'cwd': modulePath}) # run commit commands run_ret = run_commands(coms) if not run_ret: log.debug("Commit/push succeeded.") return 1 else: log.error("Commit/push failed for module %(module)s" ", branch '%(branch)s'." % {'module' : self.scmmodule, 'branch' : branch.name}) return 0 def getFiles(self, branch, ignoreWildcard=None): import os, re from turbogears import config if not ignoreWildcard: ignoreWildcard = config.get("ignore_wildcard_list") modulePath = self.canonicalPath(branchname=branch.name)['directory'] moduleFiles = [] if os.access(modulePath, os.X_OK | os.R_OK): for root, dirs, files in os.walk(modulePath, topdown=True): for name in files: filePath = os.path.join(root, name) # ignore filters if re.compile(ignoreWildcard).match(filePath): continue # allow filter if self.filefilter: if not re.compile(self.filefilter).match(filePath): continue #TODO: I don't like this check too much if self.changelog and filePath.endswith(self.changelog): log.debug("Skipping changelog file %s" % filePath) continue moduleFiles.append(filePath) # trim path from file names filesShown = [k[len(modulePath)+1:] for k in moduleFiles if k.startswith(modulePath)] filesShown.sort() return filesShown class Repository(object): """ A VCS repository. """ @classmethod def byName(cls, name): return cls.query.filter_by(name=name).one() def __str__(self): return self.name class Branch(object): """ A VCS branch. """ @classmethod def byName(cls, name): return cls.query.filter_by(name=name).one() def __str__(self): return self.name class ActionLog(object): """ Log for submits of a module """ def __str__(self): return self.summary # set up mappers between identity tables and classes mapper(Visit, visits_table) mapper(VisitIdentity, visit_identity_table, properties=dict(users=relation(User, backref='visit_identity'))) mapper(User, users_table, properties=dict(_password=users_table.c.password)) mapper(Group, groups_table, #RelatedJoin properties=dict(users=relation(User, secondary=user_group_table, backref='groups'))) mapper(Permission, permissions_table, properties=dict(groups=relation(Group, secondary=group_permission_table, backref='permissions'))) #session mapper mapper(Session, session_table) # set up mappers between tx tables and classes mapper(Module, module_table, properties=dict( branches=relation(Branch, secondary=branch_module_table, backref='modules'), logEntries=relation(ActionLog, backref='module') )) mapper(Repository, repository_table, properties=dict( modules=relation(Module, backref='repository') )) mapper(Branch, branch_table, properties=dict( logEntries=relation(ActionLog, backref='branch') )) mapper(ActionLog, action_log_table)