RepoCommand.java
/*
* Copyright (C) 2014, Google Inc. and others
*
* This program and the accompanying materials are made available under the
* terms of the Eclipse Distribution License v. 1.0 which is available at
* https://www.eclipse.org/org/documents/edl-v10.php.
*
* SPDX-License-Identifier: BSD-3-Clause
*/
package org.eclipse.jgit.gitrepo;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.StringJoiner;
import java.util.TreeMap;
import org.eclipse.jgit.annotations.NonNull;
import org.eclipse.jgit.annotations.Nullable;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.GitCommand;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.api.errors.InvalidRefNameException;
import org.eclipse.jgit.gitrepo.BareSuperprojectWriter.ExtraContent;
import org.eclipse.jgit.gitrepo.ManifestParser.IncludedFileReader;
import org.eclipse.jgit.gitrepo.internal.RepoText;
import org.eclipse.jgit.internal.JGitText;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.FileMode;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.PersonIdent;
import org.eclipse.jgit.lib.ProgressMonitor;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.RefDatabase;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.treewalk.TreeWalk;
import org.eclipse.jgit.util.FileUtils;
/**
* A class used to execute a repo command.
*
* This will parse a repo XML manifest, convert it into .gitmodules file and the
* repository config file.
*
* If called against a bare repository, it will replace all the existing content
* of the repository with the contents populated from the manifest.
*
* repo manifest allows projects overlapping, e.g. one project's manifestPath is
* "foo" and another project's manifestPath is "foo/bar". This won't
* work in git submodule, so we'll skip all the sub projects
* ("foo/bar" in the example) while converting.
*
* @see <a href="https://code.google.com/p/git-repo/">git-repo project page</a>
* @since 3.4
*/
public class RepoCommand extends GitCommand<RevCommit> {
private String manifestPath;
private String baseUri;
private URI targetUri;
private String groupsParam;
private String branch;
private String targetBranch = Constants.HEAD;
private PersonIdent author;
private RemoteReader callback;
private InputStream inputStream;
private IncludedFileReader includedReader;
private BareSuperprojectWriter.BareWriterConfig bareWriterConfig = BareSuperprojectWriter.BareWriterConfig
.getDefault();
private ProgressMonitor monitor;
private final List<ExtraContent> extraContents = new ArrayList<>();
/**
* A callback to get ref sha1 of a repository from its uri.
*
* We provided a default implementation {@link DefaultRemoteReader} to
* use ls-remote command to read the sha1 from the repository and clone the
* repository to read the file. Callers may have their own quicker
* implementation.
*
* @since 3.4
*/
public interface RemoteReader {
/**
* Read a remote ref sha1.
*
* @param uri
* The URI of the remote repository
* @param ref
* Name of the ref to lookup. May be a short-hand form, e.g.
* "master" which is automatically expanded to
* "refs/heads/master" if "refs/heads/master" already exists.
* @return the sha1 of the remote repository, or null if the ref does
* not exist.
* @throws GitAPIException
*/
@Nullable
public ObjectId sha1(String uri, String ref) throws GitAPIException;
/**
* Read a file from a remote repository.
*
* @param uri
* The URI of the remote repository
* @param ref
* The ref (branch/tag/etc.) to read
* @param path
* The relative path (inside the repo) to the file to read
* @return the file content.
* @throws GitAPIException
* @throws IOException
* @since 3.5
*
* @deprecated Use {@link #readFileWithMode(String, String, String)}
* instead
*/
@Deprecated
public default byte[] readFile(String uri, String ref, String path)
throws GitAPIException, IOException {
return readFileWithMode(uri, ref, path).getContents();
}
/**
* Read contents and mode (i.e. permissions) of the file from a remote
* repository.
*
* @param uri
* The URI of the remote repository
* @param ref
* Name of the ref to lookup. May be a short-hand form, e.g.
* "master" which is automatically expanded to
* "refs/heads/master" if "refs/heads/master" already exists.
* @param path
* The relative path (inside the repo) to the file to read
* @return The contents and file mode of the file in the given
* repository and branch. Never null.
* @throws GitAPIException
* If the ref have an invalid or ambiguous name, or it does
* not exist in the repository,
* @throws IOException
* If the object does not exist or is too large
* @since 5.2
*/
@NonNull
public RemoteFile readFileWithMode(String uri, String ref, String path)
throws GitAPIException, IOException;
}
/**
* Read-only view of contents and file mode (i.e. permissions) for a file in
* a remote repository.
*
* @since 5.2
*/
public static final class RemoteFile {
@NonNull
private final byte[] contents;
@NonNull
private final FileMode fileMode;
/**
* @param contents
* Raw contents of the file.
* @param fileMode
* Git file mode for this file (e.g. executable or regular)
*/
public RemoteFile(@NonNull byte[] contents,
@NonNull FileMode fileMode) {
this.contents = Objects.requireNonNull(contents);
this.fileMode = Objects.requireNonNull(fileMode);
}
/**
* Contents of the file.
* <p>
* Callers who receive this reference must not modify its contents (as
* it can point to internal cached data).
*
* @return Raw contents of the file. Do not modify it.
*/
@NonNull
public byte[] getContents() {
return contents;
}
/**
* @return Git file mode for this file (e.g. executable or regular)
*/
@NonNull
public FileMode getFileMode() {
return fileMode;
}
}
/** A default implementation of {@link RemoteReader} callback. */
public static class DefaultRemoteReader implements RemoteReader {
@Override
public ObjectId sha1(String uri, String ref) throws GitAPIException {
Map<String, Ref> map = Git
.lsRemoteRepository()
.setRemote(uri)
.callAsMap();
Ref r = RefDatabase.findRef(map, ref);
return r != null ? r.getObjectId() : null;
}
@Override
public RemoteFile readFileWithMode(String uri, String ref, String path)
throws GitAPIException, IOException {
File dir = FileUtils.createTempDir("jgit_", ".git", null); //$NON-NLS-1$ //$NON-NLS-2$
try (Git git = Git.cloneRepository().setBare(true).setDirectory(dir)
.setURI(uri).call()) {
Repository repo = git.getRepository();
ObjectId refCommitId = sha1(uri, ref);
if (refCommitId == null) {
throw new InvalidRefNameException(MessageFormat
.format(JGitText.get().refNotResolved, ref));
}
RevCommit commit = repo.parseCommit(refCommitId);
TreeWalk tw = TreeWalk.forPath(repo, path, commit.getTree());
// TODO(ifrade): Cope better with big files (e.g. using
// InputStream instead of byte[])
return new RemoteFile(
tw.getObjectReader().open(tw.getObjectId(0))
.getCachedBytes(Integer.MAX_VALUE),
tw.getFileMode(0));
} finally {
FileUtils.delete(dir, FileUtils.RECURSIVE);
}
}
}
@SuppressWarnings("serial")
static class ManifestErrorException extends GitAPIException {
ManifestErrorException(Throwable cause) {
super(RepoText.get().invalidManifest, cause);
}
}
@SuppressWarnings("serial")
static class RemoteUnavailableException extends GitAPIException {
RemoteUnavailableException(String uri) {
super(MessageFormat.format(RepoText.get().errorRemoteUnavailable, uri));
}
}
/**
* Constructor for RepoCommand
*
* @param repo
* the {@link org.eclipse.jgit.lib.Repository}
*/
public RepoCommand(Repository repo) {
super(repo);
}
/**
* Set path to the manifest XML file.
* <p>
* Calling {@link #setInputStream} will ignore the path set here.
*
* @param path
* (with <code>/</code> as separator)
* @return this command
*/
public RepoCommand setPath(String path) {
this.manifestPath = path;
return this;
}
/**
* Set the input stream to the manifest XML.
* <p>
* Setting inputStream will ignore the path set. It will be closed in
* {@link #call}.
*
* @param inputStream a {@link java.io.InputStream} object.
* @return this command
* @since 3.5
*/
public RepoCommand setInputStream(InputStream inputStream) {
this.inputStream = inputStream;
return this;
}
/**
* Set base URI of the paths inside the XML. This is typically the name of
* the directory holding the manifest repository, eg. for
* https://android.googlesource.com/platform/manifest, this should be
* /platform (if you would run this on android.googlesource.com) or
* https://android.googlesource.com/platform elsewhere.
*
* @param uri
* the base URI
* @return this command
*/
public RepoCommand setURI(String uri) {
this.baseUri = uri;
return this;
}
/**
* Set the URI of the superproject (this repository), so the .gitmodules
* file can specify the submodule URLs relative to the superproject.
*
* @param uri
* the URI of the repository holding the superproject.
* @return this command
* @since 4.8
*/
public RepoCommand setTargetURI(String uri) {
// The repo name is interpreted as a directory, for example
// Gerrit (http://gerrit.googlesource.com/gerrit) has a
// .gitmodules referencing ../plugins/hooks, which is
// on http://gerrit.googlesource.com/plugins/hooks,
this.targetUri = URI.create(uri + "/"); //$NON-NLS-1$
return this;
}
/**
* Set groups to sync
*
* @param groups groups separated by comma, examples: default|all|G1,-G2,-G3
* @return this command
*/
public RepoCommand setGroups(String groups) {
this.groupsParam = groups;
return this;
}
/**
* Set default branch.
* <p>
* This is generally the name of the branch the manifest file was in. If
* there's no default revision (branch) specified in manifest and no
* revision specified in project, this branch will be used.
*
* @param branch
* a branch name
* @return this command
*/
public RepoCommand setBranch(String branch) {
this.branch = branch;
return this;
}
/**
* Set target branch.
* <p>
* This is the target branch of the super project to be updated. If not set,
* default is HEAD.
* <p>
* For non-bare repositories, HEAD will always be used and this will be
* ignored.
*
* @param branch
* branch name
* @return this command
* @since 4.1
*/
public RepoCommand setTargetBranch(String branch) {
this.targetBranch = Constants.R_HEADS + branch;
return this;
}
/**
* Set whether the branch name should be recorded in .gitmodules.
* <p>
* Submodule entries in .gitmodules can include a "branch" field
* to indicate what remote branch each submodule tracks.
* <p>
* That field is used by "git submodule update --remote" to update
* to the tip of the tracked branch when asked and by Gerrit to
* update the superproject when a change on that branch is merged.
* <p>
* Subprojects that request a specific commit or tag will not have
* a branch name recorded.
* <p>
* Not implemented for non-bare repositories.
*
* @param enable Whether to record the branch name
* @return this command
* @since 4.2
*/
public RepoCommand setRecordRemoteBranch(boolean enable) {
this.bareWriterConfig.recordRemoteBranch = enable;
return this;
}
/**
* Set whether the labels field should be recorded as a label in
* .gitattributes.
* <p>
* Not implemented for non-bare repositories.
*
* @param enable Whether to record the labels in the .gitattributes
* @return this command
* @since 4.4
*/
public RepoCommand setRecordSubmoduleLabels(boolean enable) {
this.bareWriterConfig.recordSubmoduleLabels = enable;
return this;
}
/**
* Set whether the clone-depth field should be recorded as a shallow
* recommendation in .gitmodules.
* <p>
* Not implemented for non-bare repositories.
*
* @param enable Whether to record the shallow recommendation.
* @return this command
* @since 4.4
*/
public RepoCommand setRecommendShallow(boolean enable) {
this.bareWriterConfig.recordShallowSubmodules = enable;
return this;
}
/**
* The progress monitor associated with the clone operation. By default,
* this is set to <code>NullProgressMonitor</code>
*
* @see org.eclipse.jgit.lib.NullProgressMonitor
* @param monitor
* a {@link org.eclipse.jgit.lib.ProgressMonitor}
* @return this command
*/
public RepoCommand setProgressMonitor(ProgressMonitor monitor) {
this.monitor = monitor;
return this;
}
/**
* Set whether to skip projects whose commits don't exist remotely.
* <p>
* When set to true, we'll just skip the manifest entry and continue
* on to the next one.
* <p>
* When set to false (default), we'll throw an error when remote
* failures occur.
* <p>
* Not implemented for non-bare repositories.
*
* @param ignore Whether to ignore the remote failures.
* @return this command
* @since 4.3
*/
public RepoCommand setIgnoreRemoteFailures(boolean ignore) {
this.bareWriterConfig.ignoreRemoteFailures = ignore;
return this;
}
/**
* Set the author/committer for the bare repository commit.
* <p>
* For non-bare repositories, the current user will be used and this will be
* ignored.
*
* @param author
* the author's {@link org.eclipse.jgit.lib.PersonIdent}
* @return this command
*/
public RepoCommand setAuthor(PersonIdent author) {
this.author = author;
return this;
}
/**
* Set the GetHeadFromUri callback.
*
* This is only used in bare repositories.
*
* @param callback
* a {@link org.eclipse.jgit.gitrepo.RepoCommand.RemoteReader}
* object.
* @return this command
*/
public RepoCommand setRemoteReader(RemoteReader callback) {
this.callback = callback;
return this;
}
/**
* Set the IncludedFileReader callback.
*
* @param reader
* a
* {@link org.eclipse.jgit.gitrepo.ManifestParser.IncludedFileReader}
* object.
* @return this command
* @since 4.0
*/
public RepoCommand setIncludedFileReader(IncludedFileReader reader) {
this.includedReader = reader;
return this;
}
/**
* Create a file with the given content in the destination repository
*
* @param path
* where to create the file in the destination repository
* @param contents
* content for the create file
* @return this command
*
* @since 6.1
*/
public RepoCommand addToDestination(String path, String contents) {
this.extraContents.add(new ExtraContent(path, contents));
return this;
}
/** {@inheritDoc} */
@Override
public RevCommit call() throws GitAPIException {
checkCallable();
if (baseUri == null) {
baseUri = ""; //$NON-NLS-1$
}
if (inputStream == null) {
if (manifestPath == null || manifestPath.length() == 0)
throw new IllegalArgumentException(
JGitText.get().pathNotConfigured);
try {
inputStream = new FileInputStream(manifestPath);
} catch (IOException e) {
throw new IllegalArgumentException(
JGitText.get().pathNotConfigured, e);
}
}
List<RepoProject> filteredProjects;
try {
ManifestParser parser = new ManifestParser(includedReader,
manifestPath, branch, baseUri, groupsParam, repo);
parser.read(inputStream);
filteredProjects = parser.getFilteredProjects();
} catch (IOException e) {
throw new ManifestErrorException(e);
} finally {
try {
inputStream.close();
} catch (IOException e) {
// Just ignore it, it's not important.
}
}
if (repo.isBare()) {
List<RepoProject> renamedProjects = renameProjects(filteredProjects);
BareSuperprojectWriter writer = new BareSuperprojectWriter(repo, targetUri,
targetBranch,
author == null ? new PersonIdent(repo) : author,
callback == null ? new DefaultRemoteReader() : callback,
bareWriterConfig, extraContents);
return writer.write(renamedProjects);
}
RegularSuperprojectWriter writer = new RegularSuperprojectWriter(repo, monitor);
return writer.write(filteredProjects);
}
/**
* Rename the projects if there's a conflict when converted to submodules.
*
* @param projects
* parsed projects
* @return projects that are renamed if necessary
*/
private List<RepoProject> renameProjects(List<RepoProject> projects) {
Map<String, List<RepoProject>> m = new TreeMap<>();
for (RepoProject proj : projects) {
List<RepoProject> l = m.get(proj.getName());
if (l == null) {
l = new ArrayList<>();
m.put(proj.getName(), l);
}
l.add(proj);
}
List<RepoProject> ret = new ArrayList<>();
for (List<RepoProject> ps : m.values()) {
boolean nameConflict = ps.size() != 1;
for (RepoProject proj : ps) {
String name = proj.getName();
if (nameConflict) {
name += SLASH + proj.getPath();
}
RepoProject p = new RepoProject(name,
proj.getPath(), proj.getRevision(), null,
proj.getGroups(), proj.getRecommendShallow());
p.setUrl(proj.getUrl());
p.addCopyFiles(proj.getCopyFiles());
p.addLinkFiles(proj.getLinkFiles());
ret.add(p);
}
}
return ret;
}
/*
* Assume we are document "a/b/index.html", what should we put in a href to get to "a/" ?
* Returns the child if either base or child is not a bare path. This provides a missing feature in
* java.net.URI (see http://bugs.java.com/view_bug.do?bug_id=6226081).
*/
private static final String SLASH = "/"; //$NON-NLS-1$
static URI relativize(URI current, URI target) {
if (!Objects.equals(current.getHost(), target.getHost())) {
return target;
}
String cur = current.normalize().getPath();
String dest = target.normalize().getPath();
// TODO(hanwen): maybe (absolute, relative) should throw an exception.
if (cur.startsWith(SLASH) != dest.startsWith(SLASH)) {
return target;
}
while (cur.startsWith(SLASH)) {
cur = cur.substring(1);
}
while (dest.startsWith(SLASH)) {
dest = dest.substring(1);
}
if (cur.indexOf('/') == -1 || dest.indexOf('/') == -1) {
// Avoid having to special-casing in the next two ifs.
String prefix = "prefix/"; //$NON-NLS-1$
cur = prefix + cur;
dest = prefix + dest;
}
if (!cur.endsWith(SLASH)) {
// The current file doesn't matter.
int lastSlash = cur.lastIndexOf('/');
cur = cur.substring(0, lastSlash);
}
String destFile = ""; //$NON-NLS-1$
if (!dest.endsWith(SLASH)) {
// We always have to provide the destination file.
int lastSlash = dest.lastIndexOf('/');
destFile = dest.substring(lastSlash + 1, dest.length());
dest = dest.substring(0, dest.lastIndexOf('/'));
}
String[] cs = cur.split(SLASH);
String[] ds = dest.split(SLASH);
int common = 0;
while (common < cs.length && common < ds.length && cs[common].equals(ds[common])) {
common++;
}
StringJoiner j = new StringJoiner(SLASH);
for (int i = common; i < cs.length; i++) {
j.add(".."); //$NON-NLS-1$
}
for (int i = common; i < ds.length; i++) {
j.add(ds[i]);
}
j.add(destFile);
return URI.create(j.toString());
}
}