len0rd/rockbox
1
0
Fork 1
mirror of https://github.com/Rockbox/rockbox.git synced 2025-10-13 18:17:39 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

deploy: Fix various style issues.

Browse source 
Fix various pylint issues and some python3 compatibility.

Change-Id: I2c35ccc5ad2d0902c229f9de57912cf6d5a83ece
This commit is contained in:
Dominik Riebeling 2019-11-17 11:55:46 +01:00
parent aefbc13e19
commit 440791105a
5 changed files with 139 additions and 130 deletions
Download patch file Download diff file Expand all files Collapse all files

6
utils/common/deploy-rbutil.py
View file

@ -22,7 +22,7 @@ deploy.program = "RockboxUtility"
deploy.project = "rbutil/rbutilqt/rbutilqt.pro" deploy.project = "rbutil/rbutilqt/rbutilqt.pro"
deploy.svnserver = "svn://svn.rockbox.org/rockbox/" deploy.svnserver = "svn://svn.rockbox.org/rockbox/"
deploy.svnpaths = \ deploy.svnpaths = \
[ "rbutil/", ["rbutil/",
"tools/ucl", "tools/ucl",
"tools/rbspeex", "tools/rbspeex",
"utils/imxtools", "utils/imxtools",
@ -45,7 +45,7 @@ deploy.svnpaths = \
"tools/telechips.h", "tools/telechips.h",
"tools/voicefont.h", "tools/voicefont.h",
"tools/wavtrim.c", "tools/wavtrim.c",
"tools/sapi_voice.vbs" ] "tools/sapi_voice.vbs"]
deploy.useupx = False deploy.useupx = False
deploy.bundlecopy = { deploy.bundlecopy = {
"icons/rbutilqt.icns" : "Contents/Resources/", "icons/rbutilqt.icns" : "Contents/Resources/",
@ -77,7 +77,7 @@ deploy.make = {
# all files of the program. Will get put into an archive after building # all files of the program. Will get put into an archive after building
# (zip on w32, tar.bz2 on Linux). Does not apply on Mac which uses dmg. # (zip on w32, tar.bz2 on Linux). Does not apply on Mac which uses dmg.
# progexe will get added automatically. # progexe will get added automatically.
deploy.programfiles = [ ] deploy.programfiles = list()
deploy.nsisscript = "" deploy.nsisscript = ""
deploy.deploy() deploy.deploy()

6
utils/common/deploy-themeeditor.py
View file

@ -22,9 +22,9 @@ deploy.program = "rbthemeeditor"
deploy.project = "utils/themeeditor/themeeditor.pro" deploy.project = "utils/themeeditor/themeeditor.pro"
deploy.svnserver = "svn://svn.rockbox.org/rockbox/" deploy.svnserver = "svn://svn.rockbox.org/rockbox/"
deploy.svnpaths = \ deploy.svnpaths = \
[ "utils/themeeditor/", ["utils/themeeditor/",
"lib/skin_parser/", "lib/skin_parser/",
"docs/COPYING" ] "docs/COPYING"]
deploy.useupx = False deploy.useupx = False
deploy.bundlecopy = { deploy.bundlecopy = {
"resources/windowicon.icns" : "Contents/Resources/", "resources/windowicon.icns" : "Contents/Resources/",
@ -52,7 +52,7 @@ deploy.make = {
# all files of the program. Will get put into an archive after building # all files of the program. Will get put into an archive after building
# (zip on w32, tar.bz2 on Linux). Does not apply on Mac which uses dmg. # (zip on w32, tar.bz2 on Linux). Does not apply on Mac which uses dmg.
# progexe will get added automatically. # progexe will get added automatically.
deploy.programfiles = [ ] deploy.programfiles = list()
deploy.nsisscript = "utils/themeeditor/themeeditor.nsi" deploy.nsisscript = "utils/themeeditor/themeeditor.nsi"
deploy.deploy() deploy.deploy()

149
utils/common/deploy.py
View file

@ -43,21 +43,21 @@ import getopt
import time import time
import hashlib import hashlib
import tempfile import tempfile
import gitscraper
from datetime import datetime from datetime import datetime
import gitscraper
# modules that are not part of python itself. # modules that are not part of python itself.
cpus = 1 cpus = 1
try: try:
import multiprocessing import multiprocessing
cpus = multiprocessing.cpu_count() cpus = multiprocessing.cpu_count()
print "Info: %s cores found." % cpus print("Info: %s cores found." % cpus)
except ImportError: except ImportError:
print "Warning: multiprocessing module not found. Assuming 1 core." print("Warning: multiprocessing module not found. Assuming 1 core.")
# == Global stuff == # == Global stuff ==
# DLL files to ignore when searching for required DLL files. # DLL files to ignore when searching for required DLL files.
systemdlls = [ SYSTEMDLLS = [
'advapi32.dll', 'advapi32.dll',
'comdlg32.dll', 'comdlg32.dll',
'crypt32.dll', 'crypt32.dll',
@ -94,23 +94,23 @@ gitrepo = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
# == Functions == # == Functions ==
def usage(myself): def usage(myself):
print "Usage: %s [options]" % myself print("Usage: %s [options]" % myself)
print " -q, --qmake=<qmake> path to qmake" print(" -q, --qmake=<qmake> path to qmake")
print " -p, --project=<pro> path to .pro file for building with local tree" print(" -p, --project=<pro> path to .pro file for building with local tree")
print " -t, --tag=<tag> use specified tag from svn" print(" -t, --tag=<tag> use specified tag from svn")
print " -a, --add=<file> add file to build folder before building" print(" -a, --add=<file> add file to build folder before building")
print " -s, --source-only only create source archive" print(" -s, --source-only only create source archive")
print " -b, --binary-only only create binary archive" print(" -b, --binary-only only create binary archive")
if nsisscript != "": if nsisscript != "":
print " -n, --makensis=<file> path to makensis for building Windows setup program." print(" -n, --makensis=<file> path to makensis for building Windows setup program.")
if sys.platform != "darwin": if sys.platform != "darwin":
print " -d, --dynamic link dynamically instead of static" print(" -d, --dynamic link dynamically instead of static")
if sys.platform != "win32": if sys.platform != "win32":
print " -x, --cross= prefix to cross compile for win32" print(" -x, --cross= prefix to cross compile for win32")
print " -k, --keep-temp keep temporary folder on build failure" print(" -k, --keep-temp keep temporary folder on build failure")
print " -h, --help this help" print(" -h, --help this help")
print " If neither a project file nor tag is specified trunk will get downloaded" print(" If neither a project file nor tag is specified trunk will get downloaded")
print " from svn." print(" from svn.")
def which(executable): def which(executable):
@ -119,7 +119,7 @@ def which(executable):
fullpath = p + "/" + executable fullpath = p + "/" + executable
if os.path.exists(fullpath): if os.path.exists(fullpath):
return fullpath return fullpath
print "which: could not find " + executable print("which: could not find " + executable)
return "" return ""
@ -152,7 +152,7 @@ def findversion(versionfile):
def findqt(cross=""): def findqt(cross=""):
'''Search for Qt4 installation. Return path to qmake.''' '''Search for Qt4 installation. Return path to qmake.'''
print "Searching for Qt" print("Searching for Qt")
bins = [cross + "qmake", cross + "qmake-qt4"] bins = [cross + "qmake", cross + "qmake-qt4"]
for binary in bins: for binary in bins:
try: try:
@ -162,7 +162,7 @@ def findqt(cross=""):
if not result == "": if not result == "":
return result return result
except: except:
print sys.exc_value print(sys.exc_value)
return "" return ""
@ -172,27 +172,27 @@ def checkqt(qmakebin):
result = "" result = ""
# check if binary exists # check if binary exists
if not os.path.exists(qmakebin): if not os.path.exists(qmakebin):
print "Specified qmake path does not exist!" print("Specified qmake path does not exist!")
return result return result
# check version # check version
output = subprocess.Popen([qmakebin, "-version"], stdout=subprocess.PIPE, output = subprocess.Popen(
stderr=subprocess.PIPE) [qmakebin, "-version"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
cmdout = output.communicate() cmdout = output.communicate()
# don't check the qmake return code here, Qt3 doesn't return 0 on -version. # don't check the qmake return code here, Qt3 doesn't return 0 on -version.
for ou in cmdout: for ou in cmdout:
r = re.compile("Qt[^0-9]+([0-9\.]+[a-z]*)") r = re.compile("Qt[^0-9]+([0-9\.]+[a-z]*)")
m = re.search(r, ou) m = re.search(r, ou)
if not m == None: if not m == None:
print "Qt found: %s" % m.group(1) print("Qt found: %s" % m.group(1))
s = re.compile("[45]\..*") s = re.compile("[45]\..*")
n = re.search(s, m.group(1)) n = re.search(s, m.group(1))
if not n == None: if n is not None:
result = qmakebin result = qmakebin
return result return result
def qmake(qmake, projfile, platform=sys.platform, wd=".", static=True, cross=""): def qmake(qmake, projfile, platform=sys.platform, wd=".", static=True, cross=""):
print "Running qmake in %s..." % wd print("Running qmake in %s..." % wd)
command = [qmake, "-config", "release", "-config", "noccache"] command = [qmake, "-config", "release", "-config", "noccache"]
if static == True: if static == True:
command.extend(["-config", "-static"]) command.extend(["-config", "-static"])
@ -206,14 +206,14 @@ def qmake(qmake, projfile, platform=sys.platform, wd=".", static=True, cross="")
output = subprocess.Popen(command, stdout=subprocess.PIPE, cwd=wd) output = subprocess.Popen(command, stdout=subprocess.PIPE, cwd=wd)
output.communicate() output.communicate()
if not output.returncode == 0: if not output.returncode == 0:
print "qmake returned an error!" print("qmake returned an error!")
return -1 return -1
return 0 return 0
def build(wd=".", platform=sys.platform, cross=""): def build(wd=".", platform=sys.platform, cross=""):
# make # make
print "Building ..." print("Building ...")
# use the current platforms make here, cross compiling uses the native make. # use the current platforms make here, cross compiling uses the native make.
command = [make[sys.platform]] command = [make[sys.platform]]
if cpus > 1: if cpus > 1:
@ -228,36 +228,36 @@ def build(wd=".", platform=sys.platform, cross=""):
sys.stdout.write("\n") sys.stdout.write("\n")
sys.stdout.flush() sys.stdout.flush()
if not output.returncode == 0: if not output.returncode == 0:
print "Build failed!" print("Build failed!")
return -1 return -1
break break
if platform != "darwin": if platform != "darwin":
# strip. OS X handles this via macdeployqt. # strip. OS X handles this via macdeployqt.
print "Stripping binary." print("Stripping binary.")
output = subprocess.Popen([cross + "strip", progexe[platform]], \ output = subprocess.Popen([cross + "strip", progexe[platform]], \
stdout=subprocess.PIPE, cwd=wd) stdout=subprocess.PIPE, cwd=wd)
output.communicate() output.communicate()
if not output.returncode == 0: if not output.returncode == 0:
print "Stripping failed!" print("Stripping failed!")
return -1 return -1
return 0 return 0
def upxfile(wd=".", platform=sys.platform): def upxfile(wd=".", platform=sys.platform):
# run upx on binary # run upx on binary
print "UPX'ing binary ..." print("UPX'ing binary ...")
output = subprocess.Popen(["upx", progexe[platform]], \ output = subprocess.Popen(["upx", progexe[platform]], \
stdout=subprocess.PIPE, cwd=wd) stdout=subprocess.PIPE, cwd=wd)
output.communicate() output.communicate()
if not output.returncode == 0: if not output.returncode == 0:
print "UPX'ing failed!" print("UPX'ing failed!")
return -1 return -1
return 0 return 0
def runnsis(versionstring, nsis, script, srcfolder): def runnsis(versionstring, nsis, script, srcfolder):
# run script through nsis to create installer. # run script through nsis to create installer.
print "Running NSIS ..." print("Running NSIS ...")
# Assume the generated installer gets placed in the same folder the nsi # Assume the generated installer gets placed in the same folder the nsi
# script lives in. This seems to be a valid assumption unless the nsi # script lives in. This seems to be a valid assumption unless the nsi
@ -276,7 +276,7 @@ def runnsis(versionstring, nsis, script, srcfolder):
stdout=subprocess.PIPE) stdout=subprocess.PIPE)
output.communicate() output.communicate()
if not output.returncode == 0: if not output.returncode == 0:
print "NSIS failed!" print("NSIS failed!")
return -1 return -1
setupfile = program + "-" + versionstring + "-setup.exe" setupfile = program + "-" + versionstring + "-setup.exe"
# find output filename in nsis script file # find output filename in nsis script file
@ -285,7 +285,7 @@ def runnsis(versionstring, nsis, script, srcfolder):
if re.match(r'^[^;]*OutFile\s+', line) != None: if re.match(r'^[^;]*OutFile\s+', line) != None:
nsissetup = re.sub(r'^[^;]*OutFile\s+"(.+)"', r'\1', line).rstrip() nsissetup = re.sub(r'^[^;]*OutFile\s+"(.+)"', r'\1', line).rstrip()
if nsissetup == "": if nsissetup == "":
print "Could not retrieve output file name!" print("Could not retrieve output file name!")
return -1 return -1
shutil.copy(srcfolder + "/" + os.path.dirname(script) + "/" + nsissetup, \ shutil.copy(srcfolder + "/" + os.path.dirname(script) + "/" + nsissetup, \
setupfile) setupfile)
@ -330,29 +330,29 @@ def finddlls(program, extrapaths=[], cross=""):
# find DLLs in extrapaths and PATH environment variable. # find DLLs in extrapaths and PATH environment variable.
dllpaths = [] dllpaths = []
for file in dlls: for file in dlls:
if file in systemdlls: if file in SYSTEMDLLS:
print "System DLL: " + file print("System DLL: " + file)
continue continue
dllpath = "" dllpath = ""
for path in extrapaths: for path in extrapaths:
if os.path.exists(path + "/" + file): if os.path.exists(path + "/" + file):
dllpath = re.sub(r"\\", r"/", path + "/" + file) dllpath = re.sub(r"\\", r"/", path + "/" + file)
print file + ": found at " + dllpath print(file + ": found at " + dllpath)
dllpaths.append(dllpath) dllpaths.append(dllpath)
break break
if dllpath == "": if dllpath == "":
try: try:
dllpath = re.sub(r"\\", r"/", which(file)) dllpath = re.sub(r"\\", r"/", which(file))
print file + ": found at " + dllpath print(file + ": found at " + dllpath)
dllpaths.append(dllpath) dllpaths.append(dllpath)
except: except:
print "MISSING DLL: " + file print("MISSING DLL: " + file)
return dllpaths return dllpaths
def zipball(programfiles, versionstring, buildfolder, platform=sys.platform): def zipball(programfiles, versionstring, buildfolder, platform=sys.platform):
'''package created binary''' '''package created binary'''
print "Creating binary zipball." print("Creating binary zipball.")
archivebase = program + "-" + versionstring archivebase = program + "-" + versionstring
outfolder = buildfolder + "/" + archivebase outfolder = buildfolder + "/" + archivebase
archivename = archivebase + ".zip" archivename = archivebase + ".zip"
@ -379,7 +379,7 @@ def zipball(programfiles, versionstring, buildfolder, platform=sys.platform):
def tarball(programfiles, versionstring, buildfolder): def tarball(programfiles, versionstring, buildfolder):
'''package created binary''' '''package created binary'''
print "Creating binary tarball." print("Creating binary tarball.")
archivebase = program + "-" + versionstring archivebase = program + "-" + versionstring
outfolder = buildfolder + "/" + archivebase outfolder = buildfolder + "/" + archivebase
archivename = archivebase + ".tar.bz2" archivename = archivebase + ".tar.bz2"
@ -404,7 +404,7 @@ def macdeploy(versionstring, buildfolder, platform=sys.platform):
# workaround to Qt issues when building out-of-tree. Copy files into bundle. # workaround to Qt issues when building out-of-tree. Copy files into bundle.
sourcebase = buildfolder + re.sub('[^/]+.pro$', '', project) + "/" sourcebase = buildfolder + re.sub('[^/]+.pro$', '', project) + "/"
print sourcebase print(sourcebase)
for src in bundlecopy: for src in bundlecopy:
shutil.copy(sourcebase + src, appbundle + "/" + bundlecopy[src]) shutil.copy(sourcebase + src, appbundle + "/" + bundlecopy[src])
# end of Qt workaround # end of Qt workaround
@ -413,7 +413,7 @@ def macdeploy(versionstring, buildfolder, platform=sys.platform):
stdout=subprocess.PIPE, cwd=buildfolder) stdout=subprocess.PIPE, cwd=buildfolder)
output.communicate() output.communicate()
if not output.returncode == 0: if not output.returncode == 0:
print "macdeployqt failed!" print("macdeployqt failed!")
return -1 return -1
# copy dmg to output folder # copy dmg to output folder
shutil.copy(buildfolder + "/" + program + ".dmg", dmgfile) shutil.copy(buildfolder + "/" + program + ".dmg", dmgfile)
@ -440,32 +440,33 @@ def filestats(filename):
if not os.path.exists(filename): if not os.path.exists(filename):
return return
st = os.stat(filename) st = os.stat(filename)
print filename, "\n", "-" * len(filename) print(filename, "\n", "-" * len(filename))
print "Size: %i bytes" % st.st_size print("Size: %i bytes" % st.st_size)
h = filehashes(filename) h = filehashes(filename)
print "md5sum: %s" % h[0] print("md5sum: %s" % h[0])
print "sha1sum: %s" % h[1] print("sha1sum: %s" % h[1])
print "-" * len(filename), "\n" print("-" * len(filename), "\n")
def tempclean(workfolder, nopro): def tempclean(workfolder, nopro):
if nopro == True: if nopro == True:
print "Cleaning up working folder %s" % workfolder print("Cleaning up working folder %s" % workfolder)
shutil.rmtree(workfolder) shutil.rmtree(workfolder)
else: else:
print "Project file specified or cleanup disabled!" print("Project file specified or cleanup disabled!")
print "Temporary files kept at %s" % workfolder print("Temporary files kept at %s" % workfolder)
def deploy(): def deploy():
startup = time.time() startup = time.time()
try: try:
opts, args = getopt.getopt(sys.argv[1:], "q:p:t:a:n:sbdkx:i:h", opts, args = getopt.getopt(
sys.argv[1:], "q:p:t:a:n:sbdkx:i:h",
["qmake=", "project=", "tag=", "add=", "makensis=", "source-only", ["qmake=", "project=", "tag=", "add=", "makensis=", "source-only",
"binary-only", "dynamic", "keep-temp", "cross=", "buildid=", "help"]) "binary-only", "dynamic", "keep-temp", "cross=", "buildid=", "help"])
except getopt.GetoptError, err: except getopt.GetoptError as err:
print str(err) print(str(err))
usage(sys.argv[0]) usage(sys.argv[0])
sys.exit(1) sys.exit(1)
qt = "" qt = ""
@ -516,17 +517,17 @@ def deploy():
sys.exit(0) sys.exit(0)
if source == False and binary == False: if source == False and binary == False:
print "Building build neither source nor binary means nothing to do. Exiting." print("Building build neither source nor binary means nothing to do. Exiting.")
sys.exit(1) sys.exit(1)
print "Building " + progexe[platform] + " for " + platform print("Building " + progexe[platform] + " for " + platform)
# search for qmake # search for qmake
if qt == "": if qt == "":
qm = findqt(cross) qm = findqt(cross)
else: else:
qm = checkqt(qt) qm = checkqt(qt)
if qm == "": if qm == "":
print "ERROR: No suitable Qt installation found." print("ERROR: No suitable Qt installation found.")
sys.exit(1) sys.exit(1)
# create working folder. Use current directory if -p option used. # create working folder. Use current directory if -p option used.
@ -552,7 +553,7 @@ def deploy():
archivename = program + "-" + str(revision) + versionextra + "-src.tar.bz2" archivename = program + "-" + str(revision) + versionextra + "-src.tar.bz2"
ver = str(revision) ver = str(revision)
os.mkdir(sourcefolder) os.mkdir(sourcefolder)
print "Version: %s" % revision print("Version: %s" % revision)
else: else:
workfolder = "." workfolder = "."
sourcefolder = "." sourcefolder = "."
@ -561,12 +562,12 @@ def deploy():
if proj == "": if proj == "":
proj = sourcefolder + project proj = sourcefolder + project
# get sources and pack source tarball # get sources and pack source tarball
if not getsources(treehash, svnpaths, sourcefolder) == 0: if getsources(treehash, svnpaths, sourcefolder) != 0:
tempclean(workfolder, cleanup and not keeptemp) tempclean(workfolder, cleanup and not keeptemp)
sys.exit(1) sys.exit(1)
# replace version strings. # replace version strings.
print "Updating version information in sources" print("Updating version information in sources")
for f in regreplace: for f in regreplace:
infile = open(sourcefolder + "/" + f, "r") infile = open(sourcefolder + "/" + f, "r")
incontents = infile.readlines() incontents = infile.readlines()
@ -590,7 +591,7 @@ def deploy():
outfile.close() outfile.close()
if source == True: if source == True:
print "Creating source tarball %s\n" % archivename print("Creating source tarball %s\n" % archivename)
tf = tarfile.open(archivename, mode='w:bz2') tf = tarfile.open(archivename, mode='w:bz2')
tf.add(sourcefolder, os.path.basename(re.subn('/$', '', sourcefolder)[0])) tf.add(sourcefolder, os.path.basename(re.subn('/$', '', sourcefolder)[0]))
tf.close() tf.close()
@ -607,7 +608,7 @@ def deploy():
# check project file # check project file
if not os.path.exists(proj): if not os.path.exists(proj):
print "ERROR: path to project file wrong." print("ERROR: path to project file wrong.")
sys.exit(1) sys.exit(1)
# copy specified (--add) files to working folder # copy specified (--add) files to working folder
@ -615,8 +616,8 @@ def deploy():
shutil.copy(f, sourcefolder) shutil.copy(f, sourcefolder)
buildstart = time.time() buildstart = time.time()
header = "Building %s %s" % (program, ver) header = "Building %s %s" % (program, ver)
print header print(header)
print len(header) * "=" print(len(header) * "=")
# build it. # build it.
if not qmake(qm, proj, platform, sourcefolder, static, cross) == 0: if not qmake(qm, proj, platform, sourcefolder, static, cross) == 0:
@ -649,8 +650,8 @@ def deploy():
if platform == "linux2": if platform == "linux2":
for p in progfiles: for p in progfiles:
prog = sourcefolder + "/" + p prog = sourcefolder + "/" + p
output = subprocess.Popen(["file", prog], output = subprocess.Popen(
stdout=subprocess.PIPE) ["file", prog], stdout=subprocess.PIPE)
res = output.communicate() res = output.communicate()
if re.findall("ELF 64-bit", res[0]): if re.findall("ELF 64-bit", res[0]):
ver += "-64bit" ver += "-64bit"
@ -663,7 +664,7 @@ def deploy():
# display summary # display summary
headline = "Build Summary for %s" % program headline = "Build Summary for %s" % program
print "\n", headline, "\n", "=" * len(headline) print("\n", headline, "\n", "=" * len(headline))
if not archivename == "": if not archivename == "":
filestats(archivename) filestats(archivename)
filestats(archive) filestats(archive)
@ -672,10 +673,10 @@ def deploy():
dursecs = (int)(duration % 60) dursecs = (int)(duration % 60)
buildmins = (int)(buildtime / 60) buildmins = (int)(buildtime / 60)
buildsecs = (int)(buildtime % 60) buildsecs = (int)(buildtime % 60)
print "Overall time %smin %ssec, building took %smin %ssec." % \ print("Overall time %smin %ssec, building took %smin %ssec." % \
(durmins, dursecs, buildmins, buildsecs) (durmins, dursecs, buildmins, buildsecs))
if __name__ == "__main__": if __name__ == "__main__":
print "You cannot run this module directly!" print("You cannot run this module directly!")
print "Set required environment and call deploy()." print("Set required environment and call deploy().")

42
utils/common/gitscraper.py
View file

@ -42,10 +42,11 @@ def get_refs(repo):
@return Dict matching hashes to each ref. @return Dict matching hashes to each ref.
''' '''
print("Getting list of refs") print("Getting list of refs")
output = subprocess.Popen(["git", "show-ref", "--abbrev"], output = subprocess.Popen(
["git", "show-ref", "--abbrev"],
stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=repo) stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=repo)
cmdout = output.communicate() cmdout = output.communicate()
refs = {} refs = dict()
if len(cmdout[1]) > 0: if len(cmdout[1]) > 0:
print("An error occured!\n") print("An error occured!\n")
@ -53,7 +54,7 @@ def get_refs(repo):
return refs return refs
for line in cmdout: for line in cmdout:
regex = re.findall(b'([a-f0-9]+)\s+(\S+)', line) regex = re.findall(b'([a-f0-9]+)\\s+(\\S+)', line)
for r in regex: for r in regex:
# ref is the key, hash its value. # ref is the key, hash its value.
refs[r[1].decode()] = r[0].decode() refs[r[1].decode()] = r[0].decode()
@ -61,7 +62,7 @@ def get_refs(repo):
return refs return refs
def get_lstree(repo, start, filterlist=[]): def get_lstree(repo, start, filterlist=None):
'''Get recursive list of tree objects for a given tree. '''Get recursive list of tree objects for a given tree.
@param repo Path to repository root. @param repo Path to repository root.
@param start Hash identifying the tree. @param start Hash identifying the tree.
@ -69,10 +70,13 @@ def get_lstree(repo, start, filterlist=[]):
An empty list will retrieve all paths. An empty list will retrieve all paths.
@return Dict mapping filename to blob hash @return Dict mapping filename to blob hash
''' '''
output = subprocess.Popen(["git", "ls-tree", "-r", start], if filterlist is None:
filterlist = list()
output = subprocess.Popen(
["git", "ls-tree", "-r", start],
stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=repo) stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=repo)
cmdout = output.communicate() cmdout = output.communicate()
objects = {} objects = dict()
if len(cmdout[1]) > 0: if len(cmdout[1]) > 0:
print("An error occured!\n") print("An error occured!\n")
@ -80,7 +84,7 @@ def get_lstree(repo, start, filterlist=[]):
return objects return objects
for line in cmdout[0].decode().split('\n'): for line in cmdout[0].decode().split('\n'):
regex = re.findall(b'([0-9]+)\s+([a-z]+)\s+([0-9a-f]+)\s+(.*)', regex = re.findall(b'([0-9]+)\\s+([a-z]+)\\s+([0-9a-f]+)\\s+(.*)',
line.encode()) line.encode())
for rf in regex: for rf in regex:
# filter # filter
@ -121,7 +125,8 @@ def get_object(repo, blob, destfile):
@param destfile filename for blob output. @param destfile filename for blob output.
@return True if file was successfully written, False on error. @return True if file was successfully written, False on error.
''' '''
output = subprocess.Popen(["git", "cat-file", "-p", blob], output = subprocess.Popen(
["git", "cat-file", "-p", blob],
stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=repo) stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=repo)
cmdout = output.communicate() cmdout = output.communicate()
# make sure output path exists # make sure output path exists
@ -143,7 +148,8 @@ def describe_treehash(repo, treehash):
@param treehash Hash identifying the tree / commit to describe. @param treehash Hash identifying the tree / commit to describe.
@return Description string. @return Description string.
''' '''
output = subprocess.Popen(["git", "describe", treehash], output = subprocess.Popen(
["git", "describe", treehash],
stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=repo) stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=repo)
cmdout = output.communicate() cmdout = output.communicate()
if len(cmdout[1]) > 0: if len(cmdout[1]) > 0:
@ -153,7 +159,7 @@ def describe_treehash(repo, treehash):
return cmdout[0].rstrip() return cmdout[0].rstrip()
def scrape_files(repo, treehash, filelist, dest="", timestamp_files=[]): def scrape_files(repo, treehash, filelist, dest=None, timestamp_files=None):
'''Scrape list of files from repository. '''Scrape list of files from repository.
@param repo Path to repository root. @param repo Path to repository root.
@param treehash Hash identifying the tree. @param treehash Hash identifying the tree.
@ -167,7 +173,9 @@ def scrape_files(repo, treehash, filelist, dest="", timestamp_files=[]):
''' '''
print("Scraping files from repository") print("Scraping files from repository")
if dest == "": if timestamp_files is None:
timestamp_files = list()
if dest is None:
dest = tempfile.mkdtemp() dest = tempfile.mkdtemp()
treeobjects = get_lstree(repo, treehash, filelist) treeobjects = get_lstree(repo, treehash, filelist)
timestamps = {} timestamps = {}
@ -180,7 +188,7 @@ def scrape_files(repo, treehash, filelist, dest="", timestamp_files=[]):
return [dest, timestamps] return [dest, timestamps]
def archive_files(repo, treehash, filelist, basename, tmpfolder="", def archive_files(repo, treehash, filelist, basename, tmpfolder=None,
archive="tbz"): archive="tbz"):
'''Archive list of files into tarball. '''Archive list of files into tarball.
@param repo Path to repository root. @param repo Path to repository root.
@ -197,20 +205,20 @@ def archive_files(repo, treehash, filelist, basename, tmpfolder="",
@return Output filename. @return Output filename.
''' '''
if tmpfolder == "": if tmpfolder is None:
temp_remove = True temp_remove = True
tmpfolder = tempfile.mkdtemp() tmpfolder = tempfile.mkdtemp()
else: else:
temp_remove = False temp_remove = False
workfolder = scrape_files(repo, treehash, filelist, workfolder = scrape_files(
os.path.join(tmpfolder, basename))[0] repo, treehash, filelist, os.path.join(tmpfolder, basename))[0]
if basename is "": if basename is "":
return "" return ""
print("Archiving files from repository") print("Archiving files from repository")
if archive == "7z": if archive == "7z":
outfile = basename + ".7z" outfile = basename + ".7z"
output = subprocess.Popen(["7z", "a", output = subprocess.Popen(
os.path.join(os.getcwd(), basename + ".7z"), basename], ["7z", "a", os.path.join(os.getcwd(), basename + ".7z"), basename],
cwd=tmpfolder, stdout=subprocess.PIPE, stderr=subprocess.PIPE) cwd=tmpfolder, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
output.communicate() output.communicate()
elif archive == "tbz": elif archive == "tbz":

2
utils/common/tarball.py
View file

@ -1,8 +1,8 @@
#!/usr/bin/python #!/usr/bin/python
import gitscraper
import os import os
import sys import sys
import gitscraper
if len(sys.argv) < 2: if len(sys.argv) < 2:
print("Usage: %s <version|hash>" % sys.argv[0]) print("Usage: %s <version|hash>" % sys.argv[0])