Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Support an arbitrary glob, not just a suffix #9

Merged
merged 2 commits into from
Jan 8, 2016
Merged

Support an arbitrary glob, not just a suffix #9

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
fba52f5
Modify object caching for support glob path as arguments
bozaro Jan 8, 2016
6c3cd1d
Support an arbitrary glob, not just a suffix
bozaro Jan 8, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ For quick run you need:
* After unpacking archive you can run server executing:

```bash
java -jar build/deploy/git-lfs-migrate.jar -s source-repo.git -d target-repo.git -l http://test:test@lfs-server/ .psd .zip .bin
java -jar build/deploy/git-lfs-migrate.jar -s source-repo.git -d target-repo.git -l http://test:test@lfs-server/ "*.psd" "*.zip" "*.bin"
```

For example, you can convert bozaro/git-lfs-migrate to bozaro/git-lfs-migrate-converted by commands:
Expand All @@ -27,7 +27,7 @@ git clone --mirror git@github.com:bozaro/git-lfs-migrate.git

# Convert repository with moving .md and .jar file to LFS
#
# Usage: <main class> [options] LFS file suffixes
# Usage: <main class> [options] LFS file glob patterns
# Options:
# -c, --cache
# Source repository
Expand Down Expand Up @@ -56,8 +56,8 @@ java -jar git-lfs-migrate.jar \
-s git-lfs-migrate.git \
-d git-lfs-migrate-converted.git \
-g git@github.com:bozaro/git-lfs-migrate-converted.git \
.md \
.jar
"*.md" \
"*.jar"

# Push coverted repository to new repository
cd git-lfs-migrate-converted.git
Expand Down Expand Up @@ -101,5 +101,5 @@ call gradlew.bat deployZip
When build completes you can convert repository executing:

```bash
java -jar build/deploy/git-lfs-migrate.jar -s source-repo.git -d target-repo.git -l http://test:test@lfs-server/ .psd .zip .bin
java -jar build/deploy/git-lfs-migrate.jar -s source-repo.git -d target-repo.git -l http://test:test@lfs-server/ "*.psd" "*.zip" "*.bin"
```
89 changes: 57 additions & 32 deletions src/main/java/git/lfs/migrate/GitConverter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package git.lfs.migrate;

import org.apache.commons.codec.binary.Hex;
import org.eclipse.jgit.errors.InvalidPatternException;
import org.eclipse.jgit.fnmatch.FileNameMatcher;
import org.eclipse.jgit.lib.*;
import org.eclipse.jgit.revwalk.*;
import org.eclipse.jgit.treewalk.CanonicalTreeParser;
Expand Down Expand Up @@ -31,7 +33,7 @@ public class GitConverter implements AutoCloseable {
@NotNull
private static final String GIT_ATTRIBUTES = ".gitattributes";
@NotNull
private final String[] suffixes;
private final String[] globs;
@NotNull
private final File basePath;
@NotNull
Expand All @@ -41,9 +43,14 @@ public class GitConverter implements AutoCloseable {
@NotNull
private final HTreeMap<String, String> cacheSha256;

public GitConverter(@NotNull File cachePath, @NotNull File basePath, @NotNull String[] suffixes) throws IOException {
public GitConverter(@NotNull File cachePath, @NotNull File basePath, @NotNull String[] globs) throws IOException, InvalidPatternException {
this.basePath = basePath;
this.suffixes = suffixes.clone();
this.globs = globs.clone();
Arrays.sort(globs);

for (String glob : globs) {
new FileNameMatcher(glob, '/');
}

tempPath = new File(basePath, "lfs/tmp");
makeParentDirs(tempPath);
Expand Down Expand Up @@ -73,7 +80,7 @@ public ConvertTask convertTask(@NotNull ObjectReader reader, @NotNull TaskKey ke
return convertCommitTask((RevCommit) revObject);
}
if (revObject instanceof RevTree) {
return convertTreeTask(reader, revObject, false);
return convertTreeTask(reader, revObject, Objects.requireNonNull(key.getPath()));
}
if (revObject instanceof RevBlob) {
return copyTask(reader, revObject);
Expand All @@ -83,13 +90,6 @@ public ConvertTask convertTask(@NotNull ObjectReader reader, @NotNull TaskKey ke
}
throw new IllegalStateException("Unsupported object type: " + key + " (" + revObject.getClass().getName() + ")");
}
case Root: {
final RevObject revObject = new RevWalk(reader).parseAny(key.getObjectId());
if (revObject instanceof RevTree) {
return convertTreeTask(reader, revObject, true);
}
throw new IllegalStateException("Unsupported object type: " + key + " (" + revObject.getClass().getName() + ")");
}
case Attribute:
return createAttributesTask(reader, key.getObjectId());
case UploadLfs:
Expand Down Expand Up @@ -122,14 +122,14 @@ private ConvertTask convertTagTask(@NotNull RevTag revObject) throws IOException
@Override
public Iterable<TaskKey> depends() {
return Collections.singletonList(
new TaskKey(TaskType.Simple, revObject.getObject())
new TaskKey(TaskType.Simple, "", revObject.getObject())
);
}

@NotNull
@Override
public ObjectId convert(@NotNull ObjectInserter inserter, @NotNull ConvertResolver resolver, @Nullable Uploader uploader) throws IOException {
final ObjectId id = resolver.resolve(TaskType.Simple, revObject.getObject());
final ObjectId id = resolver.resolve(TaskType.Simple, "", revObject.getObject());
final TagBuilder builder = new TagBuilder();
builder.setMessage(revObject.getFullMessage());
builder.setTag(revObject.getTagName());
Expand All @@ -148,9 +148,9 @@ private ConvertTask convertCommitTask(@NotNull RevCommit revObject) throws IOExc
public Iterable<TaskKey> depends() {
List<TaskKey> result = new ArrayList<>();
for (RevCommit parent : revObject.getParents()) {
result.add(new TaskKey(TaskType.Simple, parent));
result.add(new TaskKey(TaskType.Simple, "", parent));
}
result.add(new TaskKey(TaskType.Root, revObject.getTree()));
result.add(new TaskKey(TaskType.Simple, "", revObject.getTree()));
return result;
}

Expand All @@ -164,39 +164,43 @@ public ObjectId convert(@NotNull ObjectInserter inserter, @NotNull ConvertResolv
builder.setMessage(revObject.getFullMessage());
// Set parents
for (RevCommit oldParent : revObject.getParents()) {
builder.addParentId(resolver.resolve(TaskType.Simple, oldParent));
builder.addParentId(resolver.resolve(TaskType.Simple, "", oldParent));
}
// Set tree
builder.setTreeId(resolver.resolve(TaskType.Root, revObject.getTree()));
builder.setTreeId(resolver.resolve(TaskType.Simple, "", revObject.getTree()));
return inserter.insert(builder);
}
};
}

@NotNull
private ConvertTask convertTreeTask(@NotNull ObjectReader reader, @NotNull ObjectId id, boolean rootTree) {
private ConvertTask convertTreeTask(@NotNull ObjectReader reader, @NotNull ObjectId id, @NotNull String path) {
return new ConvertTask() {
@NotNull
private List<GitTreeEntry> getEntries() throws IOException {
final List<GitTreeEntry> entries = new ArrayList<>();
final CanonicalTreeParser treeParser = new CanonicalTreeParser(null, reader, id);
boolean needAttributes = rootTree;
boolean needAttributes = path.isEmpty();
while (!treeParser.eof()) {
final FileMode fileMode = treeParser.getEntryFileMode();
final TaskType blobTask;
final String pathTask;
if (needAttributes && treeParser.getEntryPathString().equals(GIT_ATTRIBUTES)) {
blobTask = TaskType.Attribute;
pathTask = null;
needAttributes = false;
} else if (isFile(fileMode) && matchFilename(treeParser.getEntryPathString())) {
} else if (isFile(fileMode) && matchFilename(path + "/" + treeParser.getEntryPathString())) {
blobTask = TaskType.UploadLfs;
pathTask = null;
} else {
blobTask = TaskType.Simple;
pathTask = path + "/" + treeParser.getEntryPathString();
}
entries.add(new GitTreeEntry(fileMode, new TaskKey(blobTask, treeParser.getEntryObjectId()), treeParser.getEntryPathString()));
entries.add(new GitTreeEntry(fileMode, new TaskKey(blobTask, pathTask, treeParser.getEntryObjectId()), treeParser.getEntryPathString()));
treeParser.next();
}
if (needAttributes && suffixes.length > 0) {
entries.add(new GitTreeEntry(FileMode.REGULAR_FILE, new TaskKey(TaskType.Attribute, ObjectId.zeroId()), GIT_ATTRIBUTES));
if (needAttributes && globs.length > 0) {
entries.add(new GitTreeEntry(FileMode.REGULAR_FILE, new TaskKey(TaskType.Attribute, null, ObjectId.zeroId()), GIT_ATTRIBUTES));
}
return entries;
}
Expand Down Expand Up @@ -228,12 +232,21 @@ public ObjectId convert(@NotNull ObjectInserter inserter, @NotNull ConvertResolv
}

private boolean matchFilename(@NotNull String fileName) {
for (String suffix : suffixes) {
if (fileName.endsWith(suffix)) {
return true;
if (!fileName.startsWith("/")) {
throw new IllegalStateException("Unexpected file name: " + fileName);
}
try {
for (String glob : globs) {
final FileNameMatcher matcher = new FileNameMatcher(glob, null);
matcher.append(fileName.substring(1));
if (matcher.isMatch()) {
return true;
}
}
return false;
} catch (InvalidPatternException e) {
throw new IllegalArgumentException(e);
}
return false;
}

@NotNull
Expand Down Expand Up @@ -358,8 +371,8 @@ public Iterable<TaskKey> depends() throws IOException {
@Override
public ObjectId convert(@NotNull ObjectInserter inserter, @NotNull ConvertResolver resolver, @Nullable Uploader uploader) throws IOException {
final Set<String> attributes = new TreeSet<>();
for (String suffix : suffixes) {
attributes.add("*" + suffix + "\tfilter=lfs diff=lfs merge=lfs -crlf");
for (String glob : globs) {
attributes.add(glob + "\tfilter=lfs diff=lfs merge=lfs -crlf");
}
final ByteArrayOutputStream blob = new ByteArrayOutputStream();
try (BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(openAttributes(reader, id), StandardCharsets.UTF_8))) {
Expand Down Expand Up @@ -410,16 +423,28 @@ private InputStream openAttributes(@NotNull ObjectReader reader, @Nullable Objec
}

public enum TaskType {
Simple, Root, Attribute, UploadLfs,
Simple(true),
Attribute(false),
UploadLfs(false);

TaskType(boolean needPath) {
this.needPath = needPath;
}

private final boolean needPath;

public boolean needPath() {
return needPath;
}
}

public interface ConvertResolver {
@NotNull
ObjectId resolve(@NotNull TaskKey key);

@NotNull
default ObjectId resolve(@NotNull TaskType type, @NotNull ObjectId objectId) {
return resolve(new TaskKey(type, objectId));
default ObjectId resolve(@NotNull TaskType type, @Nullable String path, @NotNull ObjectId objectId) {
return resolve(new TaskKey(type, path, objectId));
}
}

Expand Down
17 changes: 9 additions & 8 deletions src/main/java/git/lfs/migrate/Main.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
import org.apache.commons.httpclient.HttpStatus;
import org.eclipse.jgit.errors.InvalidPatternException;
import org.eclipse.jgit.lib.*;
import org.eclipse.jgit.storage.file.FileRepositoryBuilder;
import org.jetbrains.annotations.NotNull;
Expand Down Expand Up @@ -43,7 +44,7 @@ public class Main {
@NotNull
private static final Logger log = LoggerFactory.getLogger(Main.class);

public static void main(@NotNull String[] args) throws IOException, InterruptedException, ExecutionException {
public static void main(@NotNull String[] args) throws IOException, InterruptedException, ExecutionException, InvalidPatternException {
final CmdArgs cmd = new CmdArgs();
final JCommander jc = new JCommander(cmd);
jc.parse(args);
Expand All @@ -69,7 +70,7 @@ public static void main(@NotNull String[] args) throws IOException, InterruptedE
}
return;
}
processRepository(cmd.src, cmd.dst, cmd.cache, auth, cmd.writeThreads, cmd.uploadThreads, cmd.suffixes.toArray(new String[cmd.suffixes.size()]));
processRepository(cmd.src, cmd.dst, cmd.cache, auth, cmd.writeThreads, cmd.uploadThreads, cmd.globs.toArray(new String[cmd.globs.size()]));
log.info("Convert time: {}", System.currentTimeMillis() - time);
}

Expand Down Expand Up @@ -112,7 +113,7 @@ private static boolean checkLfsAuthenticate(@Nullable AuthProvider auth) throws
return false;
}

public static void processRepository(@NotNull File srcPath, @NotNull File dstPath, @NotNull File cachePath, @Nullable AuthProvider auth, int writeThreads, int uploadThreads, @NotNull String... suffixes) throws IOException, InterruptedException, ExecutionException {
public static void processRepository(@NotNull File srcPath, @NotNull File dstPath, @NotNull File cachePath, @Nullable AuthProvider auth, int writeThreads, int uploadThreads, @NotNull String... globs) throws IOException, InterruptedException, ExecutionException, InvalidPatternException {
removeDirectory(dstPath);
dstPath.mkdirs();

Expand All @@ -123,7 +124,7 @@ public static void processRepository(@NotNull File srcPath, @NotNull File dstPat
.setMustExist(false)
.setGitDir(dstPath).build();

final GitConverter converter = new GitConverter(cachePath, dstPath, suffixes);
final GitConverter converter = new GitConverter(cachePath, dstPath, globs);
try {
dstRepo.create(true);
// Load all revision list.
Expand All @@ -150,7 +151,7 @@ public static void processRepository(@NotNull File srcPath, @NotNull File dstPat
for (Map.Entry<String, Ref> ref : srcRepo.getAllRefs().entrySet()) {
RefUpdate refUpdate = dstRepo.updateRef(ref.getKey());
final ObjectId oldId = ref.getValue().getObjectId();
final ObjectId newId = converted.get(new TaskKey(GitConverter.TaskType.Simple, oldId));
final ObjectId newId = converted.get(new TaskKey(GitConverter.TaskType.Simple, "", oldId));
refUpdate.setNewObjectId(newId);
refUpdate.update();
log.info(" convert ref: {} -> {} ({})", oldId.getName(), newId.getName(), ref.getKey());
Expand Down Expand Up @@ -269,7 +270,7 @@ private static SimpleDirectedGraph<TaskKey, DefaultEdge> loadTaskGraph(@NotNull
final Deque<TaskKey> queue = new ArrayDeque<>();
// Heads
for (Ref ref : refs.values()) {
final TaskKey taskKey = new TaskKey(GitConverter.TaskType.Simple, ref.getObjectId());
final TaskKey taskKey = new TaskKey(GitConverter.TaskType.Simple, "", ref.getObjectId());
if (graph.addVertex(taskKey)) {
queue.add(taskKey);
reporter.increment();
Expand Down Expand Up @@ -420,9 +421,9 @@ public static class CmdArgs {
@Parameter(names = {"--check-lfs"}, description = "Check LFS server settings and exit")
private boolean checkLfs = false;

@Parameter(description = "LFS file suffixes")
@Parameter(description = "LFS file glob patterns")
@NotNull
private List<String> suffixes = new ArrayList<>();
private List<String> globs = new ArrayList<>();
@Parameter(names = {"-h", "--help"}, description = "Show help", help = true)
private boolean help = false;
}
Expand Down
25 changes: 21 additions & 4 deletions src/main/java/git/lfs/migrate/TaskKey.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@

import org.eclipse.jgit.lib.ObjectId;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;

import java.util.Objects;

/**
* Key of converter task.
Expand All @@ -14,10 +17,16 @@ public final class TaskKey {
private final GitConverter.TaskType type;
@NotNull
private final ObjectId objectId;
@Nullable
private final String path;

public TaskKey(@NotNull GitConverter.TaskType type, @NotNull ObjectId objectId) {
public TaskKey(@NotNull GitConverter.TaskType type, @Nullable String path, @NotNull ObjectId objectId) {
this.type = type;
this.path = path;
this.objectId = objectId.copy();
if (type.needPath() == (path == null)) {
throw new IllegalStateException();
}
}

@NotNull
Expand All @@ -30,6 +39,11 @@ public ObjectId getObjectId() {
return objectId;
}

@Nullable
public String getPath() {
return path;
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
Expand All @@ -38,19 +52,22 @@ public boolean equals(Object o) {
TaskKey taskKey = (TaskKey) o;

return (type == taskKey.type)
&& objectId.equals(taskKey.objectId);

&& objectId.equals(taskKey.objectId)
&& Objects.equals(path, taskKey.path);
}

@Override
public int hashCode() {
int result = type.hashCode();
result = 31 * result + objectId.hashCode();
if (path != null) {
result = 31 * result + path.hashCode();
}
return result;
}

@Override
public String toString() {
return type + ":" + objectId.name();
return type + ":" + objectId.name() + (path == null ? "" : " (" + path + ")");
}
}