#include "repository.h"
#include <QTextStream>
#include <QDebug>
+#include <QLinkedList>
+
+static const int maxSimultaneousProcesses = 100;
+
+class ProcessCache: QLinkedList<Repository *>
+{
+public:
+ void touch(Repository *repo)
+ {
+ remove(repo);
+
+ // if the cache is too big, remove from the front
+ while (size() >= maxSimultaneousProcesses)
+ takeFirst()->closeFastImport();
+
+ // append to the end
+ append(repo);
+ }
+
+ inline void remove(Repository *repo)
+ {
+ removeOne(repo);
+ }
+};
+static ProcessCache processCache;
Repository::Repository(const Rules::Repository &rule)
- : name(rule.name)
+ : name(rule.name), commitCount(0), processHasStarted(false)
{
foreach (Rules::Repository::Branch branchRule, rule.branches) {
Branch branch;
- branch.branchFrom = branchRule.branchFrom;
- if (!branch.branchFrom.startsWith("refs/"))
- branch.branchFrom.prepend("refs/heads/");
- branch.isCreated = false;
+ branch.created = 0; // not created
branches.insert(branchRule.name, branch);
}
// create the default branch
- branches["master"].isCreated = true;
+ branches["master"].created = 1;
fastImport.setWorkingDirectory(name);
}
Repository::~Repository()
+{
+ closeFastImport();
+}
+
+void Repository::closeFastImport()
{
if (fastImport.state() != QProcess::NotRunning) {
+ fastImport.write("checkpoint\n");
+ fastImport.waitForBytesWritten(-1);
fastImport.closeWriteChannel();
- fastImport.waitForFinished();
+ if (!fastImport.waitForFinished()) {
+ fastImport.terminate();
+ if (!fastImport.waitForFinished(200))
+ qWarning() << "git-fast-import for repository" << name << "did not die";
+ }
}
+ processHasStarted = false;
+ processCache.remove(this);
}
void Repository::reloadBranches()
{
- QHash<QString, Branch>::Iterator it = branches.begin(),
- end = branches.end();
- for ( ; it != end; ++it) {
- QString branchRef = it.key();
- if (!branchRef.startsWith("refs/"))
- branchRef.prepend("refs/heads/");
-
- bool branchExists;
- // does this branch already exist?
- QProcess revParse;
- revParse.setWorkingDirectory(name);
- revParse.start("git-rev-parse", QStringList() << "--verify" << branchRef);
- revParse.waitForFinished();
-
- if (revParse.exitCode() == 0)
- branchExists = true;
- else
- branchExists = false;
-
- if (branchExists) {
- startFastImport();
- fastImport.write("reset " + branchRef.toUtf8() +
- "\nfrom " + branchRef.toUtf8() + "^0\n\n");
- it->isCreated = true;
+ QProcess revParse;
+ revParse.setWorkingDirectory(name);
+ revParse.start("git", QStringList() << "rev-parse" << "--symbolic" << "--branches");
+ revParse.waitForFinished(-1);
+
+ if (revParse.exitCode() == 0 && revParse.bytesAvailable()) {
+ while (revParse.canReadLine()) {
+ QByteArray branchName = revParse.readLine().trimmed();
+
+ //qDebug() << "Repo" << name << "reloaded branch" << branchName;
+ branches[branchName].created = 1;
+ fastImport.write("reset refs/heads/" + branchName +
+ "\nfrom refs/heads/" + branchName + "^0\n\n"
+ "progress Branch refs/heads/" + branchName + " reloaded\n");
}
}
}
void Repository::createBranch(const QString &branch, int revnum,
const QString &branchFrom, int)
{
+ startFastImport();
if (!branches.contains(branch)) {
- qCritical() << branch << "is not a known branch in repository" << name;
- exit(1);
+ qWarning() << branch << "is not a known branch in repository" << name << endl
+ << "Going to create it automatically";
}
- startFastImport();
QByteArray branchRef = branch.toUtf8();
if (!branchRef.startsWith("refs/"))
branchRef.prepend("refs/heads/");
Branch &br = branches[branch];
- if (br.isCreated) {
+ if (br.created && br.created != revnum) {
QByteArray backupBranch = branchRef + '_' + QByteArray::number(revnum);
qWarning() << branch << "already exists; backing up to" << backupBranch;
}
// now create the branch
- br.isCreated = true;
+ br.created = revnum;
QByteArray branchFromRef = branchFrom.toUtf8();
if (!branchFromRef.startsWith("refs/"))
branchFromRef.prepend("refs/heads/");
- fastImport.write("reset " + branchRef + "\nfrom " + branchFromRef + "\n\n");
+ if (!branches.contains(branchFrom) || !branches.value(branchFrom).created) {
+ qCritical() << branch << "in repository" << name
+ << "is branching from branch" << branchFrom
+ << "but the latter doesn't exist. Can't continue.";
+ exit(1);
+ }
+
+ fastImport.write("reset " + branchRef + "\nfrom " + branchFromRef + "\n\n"
+ "progress Branch " + branchRef + " created from " + branchFromRef + "\n\n");
}
Repository::Transaction *Repository::newTransaction(const QString &branch, const QString &svnprefix,
int revnum)
{
+ startFastImport();
if (!branches.contains(branch)) {
- qCritical() << branch << "is not a known branch in repository" << name;
- return 0;
+ qWarning() << branch << "is not a known branch in repository" << name << endl
+ << "Going to create it automatically";
}
Transaction *txn = new Transaction;
txn->revnum = revnum;
txn->lastmark = revnum;
- startFastImport();
+ if ((++commitCount % 10000) == 0)
+ // write everything to disk every 10000 commits
+ fastImport.write("checkpoint\n");
return txn;
}
void Repository::startFastImport()
{
if (fastImport.state() == QProcess::NotRunning) {
+ if (processHasStarted)
+ qFatal("git-fast-import has been started once and crashed?");
+ processHasStarted = true;
+
// start the process
+ QString outputFile = name;
+ outputFile.replace('/', '_');
+ outputFile.prepend("log-");
+ fastImport.setStandardOutputFile(outputFile, QIODevice::Append);
+ fastImport.setProcessChannelMode(QProcess::MergedChannels);
+
#ifndef DRY_RUN
- fastImport.setProcessChannelMode(QProcess::ForwardedChannels);
- fastImport.start("git-fast-import", QStringList());
+ fastImport.start("git", QStringList() << "fast-import");
#else
- fastImport.setStandardOutputFile(name);
fastImport.start("/bin/cat", QStringList());
#endif
+
+ reloadBranches();
}
}
QIODevice *Repository::Transaction::addFile(const QString &path, int mode, qint64 length)
{
- FileProperties fp;
- fp.mode = mode;
- fp.mark = ++lastmark;
+ int mark = ++lastmark;
+
+ if (modifiedFiles.capacity() == 0)
+ modifiedFiles.reserve(2048);
+ modifiedFiles.append("M ");
+ modifiedFiles.append(QByteArray::number(mode, 8));
+ modifiedFiles.append(" :");
+ modifiedFiles.append(QByteArray::number(mark));
+ modifiedFiles.append(' ');
+ modifiedFiles.append(path.toUtf8());
+ modifiedFiles.append("\n");
#ifndef DRY_RUN
repository->fastImport.write("blob\nmark :");
- repository->fastImport.write(QByteArray::number(fp.mark));
+ repository->fastImport.write(QByteArray::number(mark));
repository->fastImport.write("\ndata ");
repository->fastImport.write(QByteArray::number(length));
repository->fastImport.write("\n", 1);
- repository->fastImport.waitForBytesWritten(0);
#endif
- modifiedFiles.insert(path, fp);
return &repository->fastImport;
}
void Repository::Transaction::commit()
{
+ processCache.touch(repository);
+
// create the commit message
QByteArray message = log;
if (!message.endsWith('\n'))
QTextStream s(&repository->fastImport);
s << "commit " << branchRef << endl;
s << "mark :" << revnum << endl;
- s << "committer " << author << ' ' << datetime << " -0000" << endl;
+ s << "committer " << QString::fromUtf8(author) << ' ' << datetime << " -0000" << endl;
Branch &br = repository->branches[branch];
- if (!br.isCreated) {
- br.isCreated = true;
- s << "from " << br.branchFrom << endl;
+ if (!br.created) {
+ qWarning() << "Branch" << branch << "in repository" << repository->name << "doesn't exist at revision"
+ << revnum << "-- did you resume from the wrong revision?";
+ br.created = revnum;
}
s << "data " << message.length() << endl;
}
repository->fastImport.write(message);
+ repository->fastImport.putChar('\n');
// write the file deletions
- foreach (QString df, deletedFiles)
- repository->fastImport.write("D " + df.toUtf8() + "\n");
+ if (deletedFiles.contains(""))
+ repository->fastImport.write("deleteall\n");
+ else
+ foreach (QString df, deletedFiles)
+ repository->fastImport.write("D " + df.toUtf8() + "\n");
// write the file modifications
- QHash<QString, FileProperties>::ConstIterator it = modifiedFiles.constBegin();
- for ( ; it != modifiedFiles.constEnd(); ++it) {
- repository->fastImport.write("M ", 2);
- repository->fastImport.write(QByteArray::number(it->mode, 8));
- repository->fastImport.write(" :", 2);
- repository->fastImport.write(QByteArray::number(it->mark));
- repository->fastImport.write(" ", 1);
- repository->fastImport.write(it.key().toUtf8());
- repository->fastImport.write("\n", 1);
- }
-
- repository->fastImport.write("\n");
-
- while (repository->fastImport.bytesToWrite() && repository->fastImport.waitForBytesWritten()) {
- // nothing
- }
+ repository->fastImport.write(modifiedFiles);
+
+ repository->fastImport.write("\nprogress Commit #" +
+ QByteArray::number(repository->commitCount) +
+ " branch " + branch +
+ " = SVN r" + QByteArray::number(revnum) + "\n\n");
+ printf(" %d modifications to \"%s\"",
+ deletedFiles.count() + modifiedFiles.count(),
+ qPrintable(repository->name));
+
+ while (repository->fastImport.bytesToWrite())
+ if (!repository->fastImport.waitForBytesWritten(-1))
+ qFatal("Failed to write to process: %s", qPrintable(repository->fastImport.errorString()));
}
andersk / svn-all-fast-export.git / blobdiff