forked from openembedded/openembedded-core
-
Notifications
You must be signed in to change notification settings - Fork 0
/
combo-layer
executable file
·1384 lines (1257 loc) · 60.6 KB
/
combo-layer
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/env python3
# ex:ts=4:sw=4:sts=4:et
# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
#
# Copyright 2011 Intel Corporation
# Authored-by: Yu Ke <ke.yu@intel.com>
# Paul Eggleton <paul.eggleton@intel.com>
# Richard Purdie <richard.purdie@intel.com>
#
# SPDX-License-Identifier: GPL-2.0-only
#
import fnmatch
import os, sys
import optparse
import logging
import subprocess
import tempfile
import configparser
import re
import copy
import shlex
import shutil
from string import Template
from functools import reduce
__version__ = "0.2.1"
def logger_create():
logger = logging.getLogger("")
loggerhandler = logging.StreamHandler()
loggerhandler.setFormatter(logging.Formatter("[%(asctime)s] %(message)s","%H:%M:%S"))
logger.addHandler(loggerhandler)
logger.setLevel(logging.INFO)
return logger
logger = logger_create()
def get_current_branch(repodir=None):
try:
if not os.path.exists(os.path.join(repodir if repodir else '', ".git")):
# Repo not created yet (i.e. during init) so just assume master
return "master"
branchname = runcmd("git symbolic-ref HEAD 2>/dev/null", repodir).strip()
if branchname.startswith("refs/heads/"):
branchname = branchname[11:]
return branchname
except subprocess.CalledProcessError:
return ""
class Configuration(object):
"""
Manages the configuration
For an example config file, see combo-layer.conf.example
"""
def __init__(self, options):
for key, val in options.__dict__.items():
setattr(self, key, val)
def readsection(parser, section, repo):
for (name, value) in parser.items(section):
if value.startswith("@"):
self.repos[repo][name] = eval(value.strip("@"))
else:
# Apply special type transformations for some properties.
# Type matches the RawConfigParser.get*() methods.
types = {'signoff': 'boolean', 'update': 'boolean', 'history': 'boolean'}
if name in types:
value = getattr(parser, 'get' + types[name])(section, name)
self.repos[repo][name] = value
def readglobalsection(parser, section):
for (name, value) in parser.items(section):
if name == "commit_msg":
self.commit_msg_template = value
logger.debug("Loading config file %s" % self.conffile)
self.parser = configparser.ConfigParser()
with open(self.conffile) as f:
self.parser.read_file(f)
# initialize default values
self.commit_msg_template = "Automatic commit to update last_revision"
self.repos = {}
for repo in self.parser.sections():
if repo == "combo-layer-settings":
# special handling for global settings
readglobalsection(self.parser, repo)
else:
self.repos[repo] = {}
readsection(self.parser, repo, repo)
# Load local configuration, if available
self.localconffile = None
self.localparser = None
self.combobranch = None
if self.conffile.endswith('.conf'):
lcfile = self.conffile.replace('.conf', '-local.conf')
if os.path.exists(lcfile):
# Read combo layer branch
self.combobranch = get_current_branch()
logger.debug("Combo layer branch is %s" % self.combobranch)
self.localconffile = lcfile
logger.debug("Loading local config file %s" % self.localconffile)
self.localparser = configparser.ConfigParser()
with open(self.localconffile) as f:
self.localparser.readfp(f)
for section in self.localparser.sections():
if '|' in section:
sectionvals = section.split('|')
repo = sectionvals[0]
if sectionvals[1] != self.combobranch:
continue
else:
repo = section
if repo in self.repos:
readsection(self.localparser, section, repo)
def update(self, repo, option, value, initmode=False):
# If the main config has the option already, that is what we
# are expected to modify.
if self.localparser and not self.parser.has_option(repo, option):
parser = self.localparser
section = "%s|%s" % (repo, self.combobranch)
conffile = self.localconffile
if initmode and not parser.has_section(section):
parser.add_section(section)
else:
parser = self.parser
section = repo
conffile = self.conffile
parser.set(section, option, value)
with open(conffile, "w") as f:
parser.write(f)
self.repos[repo][option] = value
def sanity_check(self, initmode=False):
required_options=["src_uri", "local_repo_dir", "dest_dir", "last_revision"]
if initmode:
required_options.remove("last_revision")
msg = ""
missing_options = []
for name in self.repos:
for option in required_options:
if option not in self.repos[name]:
msg = "%s\nOption %s is not defined for component %s" %(msg, option, name)
missing_options.append(option)
# Sanitize dest_dir so that we do not have to deal with edge cases
# (unset, empty string, double slashes) in the rest of the code.
# It not being set will still be flagged as error because it is
# listed as required option above; that could be changed now.
dest_dir = os.path.normpath(self.repos[name].get("dest_dir", "."))
self.repos[name]["dest_dir"] = "." if not dest_dir else dest_dir
if msg != "":
logger.error("configuration file %s has the following error: %s" % (self.conffile,msg))
if self.localconffile and 'last_revision' in missing_options:
logger.error("local configuration file %s may be missing configuration for combo branch %s" % (self.localconffile, self.combobranch))
sys.exit(1)
# filterdiff is required by action_splitpatch, so check its availability
if subprocess.call("which filterdiff > /dev/null 2>&1", shell=True) != 0:
logger.error("ERROR: patchutils package is missing, please install it (e.g. # apt-get install patchutils)")
sys.exit(1)
def runcmd(cmd,destdir=None,printerr=True,out=None,env=None):
"""
execute command, raise CalledProcessError if fail
return output if succeed
"""
logger.debug("run cmd '%s' in %s" % (cmd, os.getcwd() if destdir is None else destdir))
if not out:
out = tempfile.TemporaryFile()
err = out
else:
err = tempfile.TemporaryFile()
try:
subprocess.check_call(cmd, stdout=out, stderr=err, cwd=destdir, shell=isinstance(cmd, str), env=env or os.environ)
except subprocess.CalledProcessError as e:
err.seek(0)
if printerr:
logger.error("%s" % err.read())
raise e
err.seek(0)
output = err.read().decode('utf-8')
logger.debug("output: %s" % output.replace(chr(0), '\\0'))
return output
def action_sync_revs(conf, args):
"""
Update the last_revision config option for each repo with the latest
revision in the remote's branch. Useful if multiple people are using
combo-layer.
"""
repos = get_repos(conf, args[1:])
for name in repos:
repo = conf.repos[name]
ldir = repo['local_repo_dir']
branch = repo.get('branch', "master")
runcmd("git fetch", ldir)
lastrev = runcmd('git rev-parse origin/%s' % branch, ldir).strip()
print("Updating %s to %s" % (name, lastrev))
conf.update(name, "last_revision", lastrev)
def action_init(conf, args):
"""
Clone component repositories
Check git is initialised; if not, copy initial data from component repos
"""
for name in conf.repos:
ldir = conf.repos[name]['local_repo_dir']
if not os.path.exists(ldir):
logger.info("cloning %s to %s" %(conf.repos[name]['src_uri'], ldir))
subprocess.check_call("git clone %s %s" % (conf.repos[name]['src_uri'], ldir), shell=True)
if not os.path.exists(".git"):
runcmd("git init")
if conf.history:
# Need a common ref for all trees.
runcmd('git commit -m "initial empty commit" --allow-empty')
startrev = runcmd('git rev-parse master').strip()
for name in conf.repos:
repo = conf.repos[name]
ldir = repo['local_repo_dir']
branch = repo.get('branch', "master")
lastrev = repo.get('last_revision', None)
if lastrev and lastrev != "HEAD":
initialrev = lastrev
if branch:
if not check_rev_branch(name, ldir, lastrev, branch):
sys.exit(1)
logger.info("Copying data from %s at specified revision %s..." % (name, lastrev))
else:
lastrev = None
initialrev = branch
logger.info("Copying data from %s..." % name)
# Sanity check initialrev and turn it into hash (required for copying history,
# because resolving a name ref only works in the component repo).
rev = runcmd('git rev-parse %s' % initialrev, ldir).strip()
if rev != initialrev:
try:
refs = runcmd('git show-ref -s %s' % initialrev, ldir).split('\n')
if len(set(refs)) > 1:
# Happens for example when configured to track
# "master" and there is a refs/heads/master. The
# traditional behavior from "git archive" (preserved
# here) it to choose the first one. This might not be
# intended, so at least warn about it.
logger.warning("%s: initial revision '%s' not unique, picking result of rev-parse = %s" %
(name, initialrev, refs[0]))
initialrev = rev
except:
# show-ref fails for hashes. Skip the sanity warning in that case.
pass
initialrev = rev
dest_dir = repo['dest_dir']
if dest_dir != ".":
extract_dir = os.path.join(os.getcwd(), dest_dir)
if not os.path.exists(extract_dir):
os.makedirs(extract_dir)
else:
extract_dir = os.getcwd()
file_filter = repo.get('file_filter', "")
exclude_patterns = repo.get('file_exclude', '').split()
def copy_selected_files(initialrev, extract_dir, file_filter, exclude_patterns, ldir,
subdir=""):
# When working inside a filtered branch which had the
# files already moved, we need to prepend the
# subdirectory to all filters, otherwise they would
# not match.
if subdir == '.':
subdir = ''
elif subdir:
subdir = os.path.normpath(subdir)
file_filter = ' '.join([subdir + '/' + x for x in file_filter.split()])
exclude_patterns = [subdir + '/' + x for x in exclude_patterns]
# To handle both cases, we cd into the target
# directory and optionally tell tar to strip the path
# prefix when the files were already moved.
subdir_components = len(subdir.split(os.path.sep)) if subdir else 0
strip=('--strip-components=%d' % subdir_components) if subdir else ''
# TODO: file_filter wild cards do not work (and haven't worked before either), because
# a) GNU tar requires a --wildcards parameter before turning on wild card matching.
# b) The semantic is not as intendend (src/*.c also matches src/foo/bar.c,
# in contrast to the other use of file_filter as parameter of "git archive"
# where it only matches .c files directly in src).
files = runcmd("git archive %s %s | tar -x -v %s -C %s %s" %
(initialrev, subdir,
strip, extract_dir, file_filter),
ldir)
if exclude_patterns:
# Implement file removal by letting tar create the
# file and then deleting it in the file system
# again. Uses the list of files created by tar (easier
# than walking the tree).
for file in files.split('\n'):
if file.endswith(os.path.sep):
continue
for pattern in exclude_patterns:
if fnmatch.fnmatch(file, pattern):
os.unlink(os.path.join(*([extract_dir] + ['..'] * subdir_components + [file])))
break
if not conf.history:
copy_selected_files(initialrev, extract_dir, file_filter, exclude_patterns, ldir)
else:
# First fetch remote history into local repository.
# We need a ref for that, so ensure that there is one.
refname = "combo-layer-init-%s" % name
runcmd("git branch -f %s %s" % (refname, initialrev), ldir)
runcmd("git fetch %s %s" % (ldir, refname))
runcmd("git branch -D %s" % refname, ldir)
# Make that the head revision.
runcmd("git checkout -b %s %s" % (name, initialrev))
# Optional: cut the history by replacing the given
# start point(s) with commits providing the same
# content (aka tree), but with commit information that
# makes it clear that this is an artifically created
# commit and nothing the original authors had anything
# to do with.
since_rev = repo.get('since_revision', '')
if since_rev:
committer = runcmd('git var GIT_AUTHOR_IDENT').strip()
# Same time stamp, no name.
author = re.sub('.* (\d+ [+-]\d+)', r'unknown <unknown> \1', committer)
logger.info('author %s' % author)
for rev in since_rev.split():
# Resolve in component repo...
rev = runcmd('git log --oneline --no-abbrev-commit -n1 %s' % rev, ldir).split()[0]
# ... and then get the tree in current
# one. The commit should be in both repos with
# the same tree, but better check here.
tree = runcmd('git show -s --pretty=format:%%T %s' % rev).strip()
with tempfile.NamedTemporaryFile(mode='wt') as editor:
editor.write('''cat >$1 <<EOF
tree %s
author %s
committer %s
%s: squashed import of component
This commit copies the entire set of files as found in
%s %s
For more information about previous commits, see the
upstream repository.
Commit created by combo-layer.
EOF
''' % (tree, author, committer, name, name, since_rev))
editor.flush()
os.environ['GIT_EDITOR'] = 'sh %s' % editor.name
runcmd('git replace --edit %s' % rev)
# Optional: rewrite history to change commit messages or to move files.
if 'hook' in repo or dest_dir != ".":
filter_branch = ['git', 'filter-branch', '--force']
with tempfile.NamedTemporaryFile(mode='wt') as hookwrapper:
if 'hook' in repo:
# Create a shell script wrapper around the original hook that
# can be used by git filter-branch. Hook may or may not have
# an absolute path.
hook = repo['hook']
hook = os.path.join(os.path.dirname(conf.conffile), '..', hook)
# The wrappers turns the commit message
# from stdin into a fake patch header.
# This is good enough for changing Subject
# and commit msg body with normal
# combo-layer hooks.
hookwrapper.write('''set -e
tmpname=$(mktemp)
trap "rm $tmpname" EXIT
echo -n 'Subject: [PATCH] ' >>$tmpname
cat >>$tmpname
if ! [ $(tail -c 1 $tmpname | od -A n -t x1) == '0a' ]; then
echo >>$tmpname
fi
echo '---' >>$tmpname
%s $tmpname $GIT_COMMIT %s
tail -c +18 $tmpname | head -c -4
''' % (hook, name))
hookwrapper.flush()
filter_branch.extend(['--msg-filter', 'bash %s' % hookwrapper.name])
if dest_dir != ".":
parent = os.path.dirname(dest_dir)
if not parent:
parent = '.'
# May run outside of the current directory, so do not assume that .git exists.
filter_branch.extend(['--tree-filter', 'mkdir -p .git/tmptree && find . -mindepth 1 -maxdepth 1 ! -name .git -print0 | xargs -0 -I SOURCE mv SOURCE .git/tmptree && mkdir -p %s && mv .git/tmptree %s' % (parent, dest_dir)])
filter_branch.append('HEAD')
runcmd(filter_branch)
runcmd('git update-ref -d refs/original/refs/heads/%s' % name)
repo['rewritten_revision'] = runcmd('git rev-parse HEAD').strip()
repo['stripped_revision'] = repo['rewritten_revision']
# Optional filter files: remove everything and re-populate using the normal filtering code.
# Override any potential .gitignore.
if file_filter or exclude_patterns:
runcmd('git rm -rf .')
if not os.path.exists(extract_dir):
os.makedirs(extract_dir)
copy_selected_files('HEAD', extract_dir, file_filter, exclude_patterns, '.',
subdir=dest_dir)
runcmd('git add --all --force .')
if runcmd('git status --porcelain'):
# Something to commit.
runcmd(['git', 'commit', '-m',
'''%s: select file subset
Files from the component repository were chosen based on
the following filters:
file_filter = %s
file_exclude = %s''' % (name, file_filter or '<empty>', repo.get('file_exclude', '<empty>'))])
repo['stripped_revision'] = runcmd('git rev-parse HEAD').strip()
if not lastrev:
lastrev = runcmd('git rev-parse %s' % initialrev, ldir).strip()
conf.update(name, "last_revision", lastrev, initmode=True)
if not conf.history:
runcmd("git add .")
else:
# Create Octopus merge commit according to http://stackoverflow.com/questions/10874149/git-octopus-merge-with-unrelated-repositoies
runcmd('git checkout master')
merge = ['git', 'merge', '--no-commit']
for name in conf.repos:
repo = conf.repos[name]
# Use branch created earlier.
merge.append(name)
# Root all commits which have no parent in the common
# ancestor in the new repository.
for start in runcmd('git log --pretty=format:%%H --max-parents=0 %s --' % name).split('\n'):
runcmd('git replace --graft %s %s' % (start, startrev))
try:
runcmd(merge)
except Exception as error:
logger.info('''Merging component repository history failed, perhaps because of merge conflicts.
It may be possible to commit anyway after resolving these conflicts.
%s''' % error)
# Create MERGE_HEAD and MERGE_MSG. "git merge" itself
# does not create MERGE_HEAD in case of a (harmless) failure,
# and we want certain auto-generated information in the
# commit message for future reference and/or automation.
with open('.git/MERGE_HEAD', 'w') as head:
with open('.git/MERGE_MSG', 'w') as msg:
msg.write('repo: initial import of components\n\n')
# head.write('%s\n' % startrev)
for name in conf.repos:
repo = conf.repos[name]
# <upstream ref> <rewritten ref> <rewritten + files removed>
msg.write('combo-layer-%s: %s %s %s\n' % (name,
repo['last_revision'],
repo['rewritten_revision'],
repo['stripped_revision']))
rev = runcmd('git rev-parse %s' % name).strip()
head.write('%s\n' % rev)
if conf.localconffile:
localadded = True
try:
runcmd("git rm --cached %s" % conf.localconffile, printerr=False)
except subprocess.CalledProcessError:
localadded = False
if localadded:
localrelpath = os.path.relpath(conf.localconffile)
runcmd("grep -q %s .gitignore || echo %s >> .gitignore" % (localrelpath, localrelpath))
runcmd("git add .gitignore")
logger.info("Added local configuration file %s to .gitignore", localrelpath)
logger.info("Initial combo layer repository data has been created; please make any changes if desired and then use 'git commit' to make the initial commit.")
else:
logger.info("Repository already initialised, nothing to do.")
def check_repo_clean(repodir):
"""
check if the repo is clean
exit if repo is dirty
"""
output=runcmd("git status --porcelain", repodir)
r = re.compile(r'\?\? patch-.*/')
dirtyout = [item for item in output.splitlines() if not r.match(item)]
if dirtyout:
logger.error("git repo %s is dirty, please fix it first", repodir)
sys.exit(1)
def check_patch(patchfile):
f = open(patchfile, 'rb')
ln = f.readline()
of = None
in_patch = False
beyond_msg = False
pre_buf = b''
while ln:
if not beyond_msg:
if ln == b'---\n':
if not of:
break
in_patch = False
beyond_msg = True
elif ln.startswith(b'--- '):
# We have a diff in the commit message
in_patch = True
if not of:
print('WARNING: %s contains a diff in its commit message, indenting to avoid failure during apply' % patchfile)
of = open(patchfile + '.tmp', 'wb')
of.write(pre_buf)
pre_buf = b''
elif in_patch and not ln[0] in b'+-@ \n\r':
in_patch = False
if of:
if in_patch:
of.write(b' ' + ln)
else:
of.write(ln)
else:
pre_buf += ln
ln = f.readline()
f.close()
if of:
of.close()
os.rename(of.name, patchfile)
def drop_to_shell(workdir=None):
if not sys.stdin.isatty():
print("Not a TTY so can't drop to shell for resolution, exiting.")
return False
shell = os.environ.get('SHELL', 'bash')
print('Dropping to shell "%s"\n' \
'When you are finished, run the following to continue:\n' \
' exit -- continue to apply the patches\n' \
' exit 1 -- abort\n' % shell);
ret = subprocess.call([shell], cwd=workdir)
if ret != 0:
print("Aborting")
return False
else:
return True
def check_rev_branch(component, repodir, rev, branch):
try:
actualbranch = runcmd("git branch --contains %s" % rev, repodir, printerr=False)
except subprocess.CalledProcessError as e:
if e.returncode == 129:
actualbranch = ""
else:
raise
if not actualbranch:
logger.error("%s: specified revision %s is invalid!" % (component, rev))
return False
branches = []
branchlist = actualbranch.split("\n")
for b in branchlist:
branches.append(b.strip().split(' ')[-1])
if branch not in branches:
logger.error("%s: specified revision %s is not on specified branch %s!" % (component, rev, branch))
return False
return True
def get_repos(conf, repo_names):
repos = []
for name in repo_names:
if name.startswith('-'):
break
else:
repos.append(name)
for repo in repos:
if not repo in conf.repos:
logger.error("Specified component '%s' not found in configuration" % repo)
sys.exit(1)
if not repos:
repos = [ repo for repo in conf.repos if conf.repos[repo].get("update", True) ]
return repos
def action_pull(conf, args):
"""
update the component repos only
"""
repos = get_repos(conf, args[1:])
# make sure all repos are clean
for name in repos:
check_repo_clean(conf.repos[name]['local_repo_dir'])
for name in repos:
repo = conf.repos[name]
ldir = repo['local_repo_dir']
branch = repo.get('branch', "master")
logger.info("update branch %s of component repo %s in %s ..." % (branch, name, ldir))
if not conf.hard_reset:
# Try to pull only the configured branch. Beware that this may fail
# when the branch is currently unknown (for example, after reconfiguring
# combo-layer). In that case we need to fetch everything and try the check out
# and pull again.
try:
runcmd("git checkout %s" % branch, ldir, printerr=False)
except subprocess.CalledProcessError:
output=runcmd("git fetch", ldir)
logger.info(output)
runcmd("git checkout %s" % branch, ldir)
runcmd("git pull --ff-only", ldir)
else:
output=runcmd("git pull --ff-only", ldir)
logger.info(output)
else:
output=runcmd("git fetch", ldir)
logger.info(output)
runcmd("git checkout %s" % branch, ldir)
runcmd("git reset --hard FETCH_HEAD", ldir)
def action_update(conf, args):
"""
update the component repos
either:
generate the patch list
apply the generated patches
or:
re-creates the entire component history and merges them
into the current branch with a merge commit
"""
components = [arg.split(':')[0] for arg in args[1:]]
revisions = {}
for arg in args[1:]:
if ':' in arg:
a = arg.split(':', 1)
revisions[a[0]] = a[1]
repos = get_repos(conf, components)
# make sure combo repo is clean
check_repo_clean(os.getcwd())
# Check whether we keep the component histories. Must be
# set either via --history command line parameter or consistently
# in combo-layer.conf. Mixing modes is (currently, and probably
# permanently because it would be complicated) not supported.
if conf.history:
history = True
else:
history = None
for name in repos:
repo = conf.repos[name]
repo_history = repo.get('history', False)
if history is None:
history = repo_history
elif history != repo_history:
logger.error("'history' property is set inconsistently")
sys.exit(1)
# Step 1: update the component repos
if conf.nopull:
logger.info("Skipping pull (-n)")
else:
action_pull(conf, ['arg0'] + components)
if history:
update_with_history(conf, components, revisions, repos)
else:
update_with_patches(conf, components, revisions, repos)
def update_with_patches(conf, components, revisions, repos):
import uuid
patch_dir = "patch-%s" % uuid.uuid4()
if not os.path.exists(patch_dir):
os.mkdir(patch_dir)
for name in repos:
revision = revisions.get(name, None)
repo = conf.repos[name]
ldir = repo['local_repo_dir']
dest_dir = repo['dest_dir']
branch = repo.get('branch', "master")
repo_patch_dir = os.path.join(os.getcwd(), patch_dir, name)
# Step 2: generate the patch list and store to patch dir
logger.info("Generating patches from %s..." % name)
top_revision = revision or branch
if not check_rev_branch(name, ldir, top_revision, branch):
sys.exit(1)
if dest_dir != ".":
prefix = "--src-prefix=a/%s/ --dst-prefix=b/%s/" % (dest_dir, dest_dir)
else:
prefix = ""
if repo['last_revision'] == "":
logger.info("Warning: last_revision of component %s is not set, starting from the first commit" % name)
patch_cmd_range = "--root %s" % top_revision
rev_cmd_range = top_revision
else:
if not check_rev_branch(name, ldir, repo['last_revision'], branch):
sys.exit(1)
patch_cmd_range = "%s..%s" % (repo['last_revision'], top_revision)
rev_cmd_range = patch_cmd_range
file_filter = repo.get('file_filter',".")
# Filter out unwanted files
exclude = repo.get('file_exclude', '')
if exclude:
for path in exclude.split():
p = "%s/%s" % (dest_dir, path) if dest_dir != '.' else path
file_filter += " ':!%s'" % p
patch_cmd = "git format-patch -N %s --output-directory %s %s -- %s" % \
(prefix,repo_patch_dir, patch_cmd_range, file_filter)
output = runcmd(patch_cmd, ldir)
logger.debug("generated patch set:\n%s" % output)
patchlist = output.splitlines()
rev_cmd = "git rev-list --no-merges %s -- %s" % (rev_cmd_range, file_filter)
revlist = runcmd(rev_cmd, ldir).splitlines()
# Step 3: Call repo specific hook to adjust patch
if 'hook' in repo:
# hook parameter is: ./hook patchpath revision reponame
count=len(revlist)-1
for patch in patchlist:
runcmd("%s %s %s %s" % (repo['hook'], patch, revlist[count], name))
count=count-1
# Step 4: write patch list and revision list to file, for user to edit later
patchlist_file = os.path.join(os.getcwd(), patch_dir, "patchlist-%s" % name)
repo['patchlist'] = patchlist_file
f = open(patchlist_file, 'w')
count=len(revlist)-1
for patch in patchlist:
f.write("%s %s\n" % (patch, revlist[count]))
check_patch(os.path.join(patch_dir, patch))
count=count-1
f.close()
# Step 5: invoke bash for user to edit patch and patch list
if conf.interactive:
print('You may now edit the patch and patch list in %s\n' \
'For example, you can remove unwanted patch entries from patchlist-*, so that they will be not applied later' % patch_dir);
if not drop_to_shell(patch_dir):
sys.exit(1)
# Step 6: apply the generated and revised patch
apply_patchlist(conf, repos)
runcmd("rm -rf %s" % patch_dir)
# Step 7: commit the updated config file if it's being tracked
commit_conf_file(conf, components)
def conf_commit_msg(conf, components):
# create the "components" string
component_str = "all components"
if len(components) > 0:
# otherwise tell which components were actually changed
component_str = ", ".join(components)
# expand the template with known values
template = Template(conf.commit_msg_template)
msg = template.substitute(components = component_str)
return msg
def commit_conf_file(conf, components, commit=True):
relpath = os.path.relpath(conf.conffile)
try:
output = runcmd("git status --porcelain %s" % relpath, printerr=False)
except:
# Outside the repository
output = None
if output:
if output.lstrip().startswith("M"):
logger.info("Committing updated configuration file")
if commit:
msg = conf_commit_msg(conf, components)
runcmd('git commit -m'.split() + [msg, relpath])
else:
runcmd('git add %s' % relpath)
return True
return False
def apply_patchlist(conf, repos):
"""
apply the generated patch list to combo repo
"""
for name in repos:
repo = conf.repos[name]
lastrev = repo["last_revision"]
prevrev = lastrev
# Get non-blank lines from patch list file
patchlist = []
if os.path.exists(repo['patchlist']) or not conf.interactive:
# Note: we want this to fail here if the file doesn't exist and we're not in
# interactive mode since the file should exist in this case
with open(repo['patchlist']) as f:
for line in f:
line = line.rstrip()
if line:
patchlist.append(line)
ldir = conf.repos[name]['local_repo_dir']
branch = conf.repos[name].get('branch', "master")
branchrev = runcmd("git rev-parse %s" % branch, ldir).strip()
if patchlist:
logger.info("Applying patches from %s..." % name)
linecount = len(patchlist)
i = 1
for line in patchlist:
patchfile = line.split()[0]
lastrev = line.split()[1]
patchdisp = os.path.relpath(patchfile)
if os.path.getsize(patchfile) == 0:
logger.info("(skipping %d/%d %s - no changes)" % (i, linecount, patchdisp))
else:
cmd = "git am --keep-cr %s-p1 %s" % ('-s ' if repo.get('signoff', True) else '', patchfile)
logger.info("Applying %d/%d: %s" % (i, linecount, patchdisp))
try:
runcmd(cmd)
except subprocess.CalledProcessError:
logger.info('Running "git am --abort" to cleanup repo')
runcmd("git am --abort")
logger.error('"%s" failed' % cmd)
logger.info("Please manually apply patch %s" % patchdisp)
logger.info("Note: if you exit and continue applying without manually applying the patch, it will be skipped")
if not drop_to_shell():
if prevrev != repo['last_revision']:
conf.update(name, "last_revision", prevrev)
sys.exit(1)
prevrev = lastrev
i += 1
# Once all patches are applied, we should update
# last_revision to the branch head instead of the last
# applied patch. The two are not necessarily the same when
# the last commit is a merge commit or when the patches at
# the branch head were intentionally excluded.
#
# If we do not do that for a merge commit, the next
# combo-layer run will only exclude patches reachable from
# one of the merged branches and try to re-apply patches
# from other branches even though they were already
# copied.
#
# If patches were intentionally excluded, the next run will
# present them again instead of skipping over them. This
# may or may not be intended, so the code here is conservative
# and only addresses the "head is merge commit" case.
if lastrev != branchrev and \
len(runcmd("git show --pretty=format:%%P --no-patch %s" % branch, ldir).split()) > 1:
lastrev = branchrev
else:
logger.info("No patches to apply from %s" % name)
lastrev = branchrev
if lastrev != repo['last_revision']:
conf.update(name, "last_revision", lastrev)
def action_splitpatch(conf, args):
"""
generate the commit patch and
split the patch per repo
"""
logger.debug("action_splitpatch")
if len(args) > 1:
commit = args[1]
else:
commit = "HEAD"
patchdir = "splitpatch-%s" % commit
if not os.path.exists(patchdir):
os.mkdir(patchdir)
# filerange_root is for the repo whose dest_dir is root "."
# and it should be specified by excluding all other repo dest dir
# like "-x repo1 -x repo2 -x repo3 ..."
filerange_root = ""
for name in conf.repos:
dest_dir = conf.repos[name]['dest_dir']
if dest_dir != ".":
filerange_root = '%s -x "%s/*"' % (filerange_root, dest_dir)
for name in conf.repos:
dest_dir = conf.repos[name]['dest_dir']
patch_filename = "%s/%s.patch" % (patchdir, name)
if dest_dir == ".":
cmd = "git format-patch -n1 --stdout %s^..%s | filterdiff -p1 %s > %s" % (commit, commit, filerange_root, patch_filename)
else:
cmd = "git format-patch --no-prefix -n1 --stdout %s^..%s -- %s > %s" % (commit, commit, dest_dir, patch_filename)
runcmd(cmd)
# Detect empty patches (including those produced by filterdiff above
# that contain only preamble text)
if os.path.getsize(patch_filename) == 0 or runcmd("filterdiff %s" % patch_filename) == "":
os.remove(patch_filename)
logger.info("(skipping %s - no changes)", name)
else:
logger.info(patch_filename)
def update_with_history(conf, components, revisions, repos):
'''Update all components with full history.
Works by importing all commits reachable from a component's
current head revision. If those commits are rooted in an already
imported commit, their content gets mixed with the content of the
combined repo of that commit (new or modified files overwritten,
removed files removed).
The last commit is an artificial merge commit that merges all the
updated components into the combined repository.
The HEAD ref only gets updated at the very end. All intermediate work
happens in a worktree which will get garbage collected by git eventually
after a failure.
'''
# Remember current HEAD and what we need to add to it.
head = runcmd("git rev-parse HEAD").strip()
additional_heads = {}
# Track the mapping between original commit and commit in the
# combined repo. We do not have to distinguish between components,
# because commit hashes are different anyway. Often we can
# skip find_revs() entirely (for example, when all new commits
# are derived from the last imported revision).
#
# Using "head" (typically the merge commit) instead of the actual
# commit for the component leads to a nicer history in the combined
# repo.
old2new_revs = {}
for name in repos:
repo = conf.repos[name]
revision = repo['last_revision']
if revision:
old2new_revs[revision] = head
def add_p(parents):
'''Insert -p before each entry.'''
parameters = []
for p in parents:
parameters.append('-p')
parameters.append(p)
return parameters
# Do all intermediate work with a separate work dir and index,
# chosen via env variables (can't use "git worktree", it is too
# new). This is useful (no changes to current work tree unless the
# update succeeds) and required (otherwise we end up temporarily
# removing the combo-layer hooks that we currently use when
# importing a new component).
#
# Not cleaned up after a failure at the moment.
wdir = os.path.join(os.getcwd(), ".git", "combo-layer")
windex = wdir + ".index"
if os.path.isdir(wdir):
shutil.rmtree(wdir)
os.mkdir(wdir)
wenv = copy.deepcopy(os.environ)
wenv["GIT_WORK_TREE"] = wdir
wenv["GIT_INDEX_FILE"] = windex
# This one turned out to be needed in practice.
wenv["GIT_OBJECT_DIRECTORY"] = os.path.join(os.getcwd(), ".git", "objects")
wargs = {"destdir": wdir, "env": wenv}
for name in repos:
revision = revisions.get(name, None)
repo = conf.repos[name]
ldir = repo['local_repo_dir']
dest_dir = repo['dest_dir']
branch = repo.get('branch', "master")
hook = repo.get('hook', None)
largs = {"destdir": ldir, "env": None}
file_include = repo.get('file_filter', '').split()
file_include.sort() # make sure that short entries like '.' come first.
file_exclude = repo.get('file_exclude', '').split()
def include_file(file):
if not file_include:
# No explicit filter set, include file.
return True
for filter in file_include:
if filter == '.':
# Another special case: include current directory and thus all files.
return True
if os.path.commonprefix((filter, file)) == filter:
# Included in directory or direct file match.
return True
# Check for wildcard match *with* allowing * to match /, i.e.
# src/*.c does match src/foobar/*.c. That's not how it is done elsewhere
# when passing the filtering to "git archive", but it is unclear what
# the intended semantic is (the comment on file_exclude that "append a * wildcard
# at the end" to match the full content of a directories implies that
# slashes are indeed not special), so here we simply do what's easy to
# implement in Python.
logger.debug('fnmatch(%s, %s)' % (file, filter))
if fnmatch.fnmatchcase(file, filter):
return True
return False
def exclude_file(file):