forked from len0rd/rockbox
91cf4a7768
When scraping files from the repository allow passing an additional list of files for which the timestamp of the last change is retrieved. Since this is slow (the log needs to be traversed, since a blob doesn't know about commits its referenced by) only specified files get this retrieved. Change-Id: I61a2ef812ac18ffb5898c01ce070046d4ca00a15
227 lines
7.9 KiB
Python
Executable file
227 lines
7.9 KiB
Python
Executable file
#!/usr/bin/python
|
|
# __________ __ ___.
|
|
# Open \______ \ ____ ____ | | _\_ |__ _______ ___
|
|
# Source | _// _ \_/ ___\| |/ /| __ \ / _ \ \/ /
|
|
# Jukebox | | ( <_> ) \___| < | \_\ ( <_> > < <
|
|
# Firmware |____|_ /\____/ \___ >__|_ \|___ /\____/__/\_ \
|
|
# \/ \/ \/ \/ \/
|
|
#
|
|
# Copyright (c) 2012 Dominik Riebeling
|
|
#
|
|
# All files in this archive are subject to the GNU General Public License.
|
|
# See the file COPYING in the source tree root for full license agreement.
|
|
#
|
|
# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
|
|
# KIND, either express or implied.
|
|
#
|
|
|
|
'''Scrape files from a git repository.
|
|
|
|
This module provides functions to get a subset of files from a git repository.
|
|
The files to retrieve can be specified, and the git tree to work on can be
|
|
specified. That was arbitrary trees can be retrieved (like a subset of files
|
|
for a given tag).
|
|
|
|
Retrieved files can be packaged into a bzip2 compressed tarball or stored in a
|
|
given folder for processing afterwards.
|
|
|
|
Calls git commands directly for maximum compatibility.
|
|
'''
|
|
|
|
import re
|
|
import subprocess
|
|
import os
|
|
import tarfile
|
|
import tempfile
|
|
import shutil
|
|
|
|
|
|
def get_refs(repo):
|
|
'''Get dict matching refs to hashes from repository pointed to by repo.
|
|
@param repo Path to repository root.
|
|
@return Dict matching hashes to each ref.
|
|
'''
|
|
print("Getting list of refs")
|
|
output = subprocess.Popen(["git", "show-ref", "--abbrev"],
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=repo)
|
|
cmdout = output.communicate()
|
|
refs = {}
|
|
|
|
if len(cmdout[1]) > 0:
|
|
print("An error occured!\n")
|
|
print(cmdout[1])
|
|
return refs
|
|
|
|
for line in cmdout:
|
|
regex = re.findall(b'([a-f0-9]+)\s+(\S+)', line)
|
|
for r in regex:
|
|
# ref is the key, hash its value.
|
|
refs[r[1].decode()] = r[0].decode()
|
|
|
|
return refs
|
|
|
|
|
|
def get_lstree(repo, start, filterlist=[]):
|
|
'''Get recursive list of tree objects for a given tree.
|
|
@param repo Path to repository root.
|
|
@param start Hash identifying the tree.
|
|
@param filterlist List of paths to retrieve objecs hashes for.
|
|
An empty list will retrieve all paths.
|
|
@return Dict mapping filename to blob hash
|
|
'''
|
|
output = subprocess.Popen(["git", "ls-tree", "-r", start],
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=repo)
|
|
cmdout = output.communicate()
|
|
objects = {}
|
|
|
|
if len(cmdout[1]) > 0:
|
|
print("An error occured!\n")
|
|
print(cmdout[1])
|
|
return objects
|
|
|
|
for line in cmdout[0].decode().split('\n'):
|
|
regex = re.findall(b'([0-9]+)\s+([a-z]+)\s+([0-9a-f]+)\s+(\S+)',
|
|
line.encode())
|
|
for rf in regex:
|
|
# filter
|
|
add = False
|
|
for f in filterlist:
|
|
if rf[3].decode().find(f) == 0:
|
|
add = True
|
|
|
|
# If two files have the same content they have the same hash, so
|
|
# the filename has to be used as key.
|
|
if len(filterlist) == 0 or add == True:
|
|
if rf[3] in objects:
|
|
print("FATAL: key already exists in dict!")
|
|
return {}
|
|
objects[rf[3]] = rf[2]
|
|
return objects
|
|
|
|
|
|
def get_file_timestamp(repo, tree, filename):
|
|
'''Get timestamp for a file.
|
|
@param repo Path to repository root.
|
|
@param tree Hash of tree to use.
|
|
@param filename Filename in tree
|
|
@return Timestamp as string.
|
|
'''
|
|
output = subprocess.Popen(
|
|
["git", "log", "--format=%ai", "-n", "1", tree, filename],
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=repo)
|
|
cmdout = output.communicate()
|
|
|
|
return cmdout[0].decode().rstrip()
|
|
|
|
|
|
def get_object(repo, blob, destfile):
|
|
'''Get an identified object from the repository.
|
|
@param repo Path to repository root.
|
|
@param blob hash for blob to retrieve.
|
|
@param destfile filename for blob output.
|
|
@return True if file was successfully written, False on error.
|
|
'''
|
|
output = subprocess.Popen(["git", "cat-file", "-p", blob],
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=repo)
|
|
cmdout = output.communicate()
|
|
# make sure output path exists
|
|
if len(cmdout[1]) > 0:
|
|
print("An error occured!\n")
|
|
print(cmdout[1])
|
|
return False
|
|
if not os.path.exists(os.path.dirname(destfile)):
|
|
os.makedirs(os.path.dirname(destfile))
|
|
f = open(destfile, 'wb')
|
|
f.write(cmdout[0])
|
|
f.close()
|
|
return True
|
|
|
|
|
|
def describe_treehash(repo, treehash):
|
|
'''Retrieve output of git-describe for a given hash.
|
|
@param repo Path to repository root.
|
|
@param treehash Hash identifying the tree / commit to describe.
|
|
@return Description string.
|
|
'''
|
|
output = subprocess.Popen(["git", "describe", treehash],
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=repo)
|
|
cmdout = output.communicate()
|
|
if len(cmdout[1]) > 0:
|
|
print("An error occured!\n")
|
|
print(cmdout[1])
|
|
return ""
|
|
return cmdout[0].rstrip()
|
|
|
|
|
|
def scrape_files(repo, treehash, filelist, dest="", timestamp_files=[]):
|
|
'''Scrape list of files from repository.
|
|
@param repo Path to repository root.
|
|
@param treehash Hash identifying the tree.
|
|
@param filelist List of files to get from repository.
|
|
@param dest Destination path for files. Files will get retrieved with full
|
|
path from the repository, and the folder structure will get
|
|
created below dest as necessary.
|
|
@param timestamp_files List of files to also get the last modified date.
|
|
WARNING: this is SLOW!
|
|
@return Destination path, filename:timestamp dict.
|
|
'''
|
|
print("Scraping files from repository")
|
|
|
|
if dest == "":
|
|
dest = tempfile.mkdtemp()
|
|
treeobjects = get_lstree(repo, treehash, filelist)
|
|
timestamps = {}
|
|
for obj in treeobjects:
|
|
get_object(repo, treeobjects[obj], os.path.join(dest.encode(), obj))
|
|
for f in timestamp_files:
|
|
if obj.find(f) == 0:
|
|
timestamps[obj] = get_file_timestamp(repo, treehash, obj)
|
|
|
|
return [dest, timestamps]
|
|
|
|
|
|
def archive_files(repo, treehash, filelist, basename, tmpfolder="",
|
|
archive="tbz"):
|
|
'''Archive list of files into tarball.
|
|
@param repo Path to repository root.
|
|
@param treehash Hash identifying the tree.
|
|
@param filelist List of files to archive. All files in the archive if left
|
|
empty.
|
|
@param basename Basename (including path) of output file. Will get used as
|
|
basename inside of the archive as well (i.e. no tarbomb).
|
|
@param tmpfolder Folder to put intermediate files in. If no folder is given
|
|
a temporary one will get used.
|
|
@param archive Type of archive to create. Supported values are "tbz" and
|
|
"7z". The latter requires the 7z binary available in the
|
|
system's path.
|
|
@return Output filename.
|
|
'''
|
|
|
|
if tmpfolder == "":
|
|
temp_remove = True
|
|
tmpfolder = tempfile.mkdtemp()
|
|
else:
|
|
temp_remove = False
|
|
workfolder = scrape_files(repo, treehash, filelist,
|
|
os.path.join(tmpfolder, basename))[0]
|
|
if basename is "":
|
|
return ""
|
|
print("Archiving files from repository")
|
|
if archive == "7z":
|
|
outfile = basename + ".7z"
|
|
output = subprocess.Popen(["7z", "a",
|
|
os.path.join(os.getcwd(), basename + ".7z"), basename],
|
|
cwd=tmpfolder, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
output.communicate()
|
|
elif archive == "tbz":
|
|
outfile = basename + ".tar.bz2"
|
|
tf = tarfile.open(outfile, "w:bz2")
|
|
tf.add(workfolder, basename)
|
|
tf.close()
|
|
else:
|
|
print("Files not archived")
|
|
if tmpfolder != workfolder:
|
|
shutil.rmtree(workfolder)
|
|
if temp_remove:
|
|
shutil.rmtree(tmpfolder)
|
|
return outfile
|