Add a --no-metadata option to suppress the svn info in commit messages.

[svn-all-fast-export.git] / src / repository.cpp

diff --git a/src/repository.cpp b/src/repository.cpp
index 7c085ceabce94c16c3a10dd88822ccaaf7ee9158..9c2a9d5421964cd2261a0f75c2ebedf9ae91d369 100644 (file)
--- a/src/repository.cpp
+++ b/src/repository.cpp
@@ -16,11 +16,37 @@
  */
 
 #include "repository.h"
+#include "options.h"
 #include <QTextStream>
 #include <QDebug>
+#include <QLinkedList>
+
+static const int maxSimultaneousProcesses = 100;
+
+class ProcessCache: QLinkedList<Repository *>
+{
+public:
+    void touch(Repository *repo)
+    {
+        remove(repo);
+
+        // if the cache is too big, remove from the front
+        while (size() >= maxSimultaneousProcesses)
+            takeFirst()->closeFastImport();
+
+        // append to the end
+        append(repo);
+    }
+
+    inline void remove(Repository *repo)
+    {
+        removeOne(repo);
+    }
+};
+static ProcessCache processCache;
 
 Repository::Repository(const Rules::Repository &rule)
-    : name(rule.name), processHasStarted(false)
+    : name(rule.name), commitCount(0), outstandingTransactions(0), processHasStarted(false)
 {
     foreach (Rules::Repository::Branch branchRule, rule.branches) {
         Branch branch;
@@ -36,11 +62,25 @@ Repository::Repository(const Rules::Repository &rule)
 }
 
 Repository::~Repository()
+{
+    Q_ASSERT(outstandingTransactions == 0);
+    closeFastImport();
+}
+
+void Repository::closeFastImport()
 {
     if (fastImport.state() != QProcess::NotRunning) {
+        fastImport.write("checkpoint\n");
+        fastImport.waitForBytesWritten(-1);
         fastImport.closeWriteChannel();
-        fastImport.waitForFinished();
+        if (!fastImport.waitForFinished()) {
+            fastImport.terminate();
+            if (!fastImport.waitForFinished(200))
+                qWarning() << "git-fast-import for repository" << name << "did not die";
+        }
     }
+    processHasStarted = false;
+    processCache.remove(this);
 }
 
 void Repository::reloadBranches()
@@ -48,17 +88,17 @@ void Repository::reloadBranches()
     QProcess revParse;
     revParse.setWorkingDirectory(name);
     revParse.start("git", QStringList() << "rev-parse" << "--symbolic" << "--branches");
-    revParse.waitForFinished();
+    revParse.waitForFinished(-1);
 
     if (revParse.exitCode() == 0 && revParse.bytesAvailable()) {
-        startFastImport();
-
         while (revParse.canReadLine()) {
-            QByteArray branchName = revParse.readLine();
+            QByteArray branchName = revParse.readLine().trimmed();
 
+            //qDebug() << "Repo" << name << "reloaded branch" << branchName;
             branches[branchName].created = 1;
             fastImport.write("reset refs/heads/" + branchName +
-                             "\nfrom refs/heads/" + branchName + "^0\n\n");
+                             "\nfrom refs/heads/" + branchName + "^0\n\n"
+                             "progress Branch refs/heads/" + branchName + " reloaded\n");
         }
     }
 }
@@ -66,12 +106,12 @@ void Repository::reloadBranches()
 void Repository::createBranch(const QString &branch, int revnum,
                               const QString &branchFrom, int)
 {
+    startFastImport();
     if (!branches.contains(branch)) {
         qWarning() << branch << "is not a known branch in repository" << name << endl
                    << "Going to create it automatically";
     }
 
-    startFastImport();
     QByteArray branchRef = branch.toUtf8();
     if (!branchRef.startsWith("refs/"))
         branchRef.prepend("refs/heads/");
@@ -97,12 +137,14 @@ void Repository::createBranch(const QString &branch, int revnum,
         exit(1);
     }
 
-    fastImport.write("reset " + branchRef + "\nfrom " + branchFromRef + "\n\n");
+    fastImport.write("reset " + branchRef + "\nfrom " + branchFromRef + "\n\n"
+        "progress Branch " + branchRef + " created from " + branchFromRef + "\n\n");
 }
 
 Repository::Transaction *Repository::newTransaction(const QString &branch, const QString &svnprefix,
                                                     int revnum)
 {
+    startFastImport();
     if (!branches.contains(branch)) {
         qWarning() << branch << "is not a known branch in repository" << name << endl
                    << "Going to create it automatically";
@@ -114,12 +156,12 @@ Repository::Transaction *Repository::newTransaction(const QString &branch, const
     txn->svnprefix = svnprefix.toUtf8();
     txn->datetime = 0;
     txn->revnum = revnum;
-    txn->lastmark = revnum;
 
-    startFastImport();
     if ((++commitCount % 10000) == 0)
         // write everything to disk every 10000 commits
         fastImport.write("checkpoint\n");
+    if (outstandingTransactions++ == 0)
+        lastmark = 1;           // reset the mark number
     return txn;
 }
 
@@ -138,15 +180,18 @@ void Repository::startFastImport()
         fastImport.setProcessChannelMode(QProcess::MergedChannels);
 
 #ifndef DRY_RUN
-        fastImport.start("git-fast-import", QStringList());
+        fastImport.start("git", QStringList() << "fast-import");
 #else
         fastImport.start("/bin/cat", QStringList());
 #endif
+
+        reloadBranches();
     }
 }
 
 Repository::Transaction::~Transaction()
 {
+    --repository->outstandingTransactions;
 }
 
 void Repository::Transaction::setAuthor(const QByteArray &a)
@@ -171,29 +216,39 @@ void Repository::Transaction::deleteFile(const QString &path)
 
 QIODevice *Repository::Transaction::addFile(const QString &path, int mode, qint64 length)
 {
-    FileProperties fp;
-    fp.mode = mode;
-    fp.mark = ++lastmark;
+    int mark = ++repository->lastmark;
+
+    if (modifiedFiles.capacity() == 0)
+        modifiedFiles.reserve(2048);
+    modifiedFiles.append("M ");
+    modifiedFiles.append(QByteArray::number(mode, 8));
+    modifiedFiles.append(" :");
+    modifiedFiles.append(QByteArray::number(mark));
+    modifiedFiles.append(' ');
+    modifiedFiles.append(path.toUtf8());
+    modifiedFiles.append("\n");
 
 #ifndef DRY_RUN
     repository->fastImport.write("blob\nmark :");
-    repository->fastImport.write(QByteArray::number(fp.mark));
+    repository->fastImport.write(QByteArray::number(mark));
     repository->fastImport.write("\ndata ");
     repository->fastImport.write(QByteArray::number(length));
     repository->fastImport.write("\n", 1);
 #endif
 
-    modifiedFiles.insert(path, fp);
     return &repository->fastImport;
 }
 
 void Repository::Transaction::commit()
 {
+    processCache.touch(repository);
+
     // create the commit message
     QByteArray message = log;
     if (!message.endsWith('\n'))
         message += '\n';
-    message += "\nsvn path=" + svnprefix + "; revision=" + QByteArray::number(revnum) + "\n";
+    if (Options::globalOptions->switches.value("metadata", true))
+        message += "\nsvn path=" + svnprefix + "; revision=" + QByteArray::number(revnum) + "\n";
 
     {
         QByteArray branchRef = branch;
@@ -202,7 +257,6 @@ void Repository::Transaction::commit()
 
         QTextStream s(&repository->fastImport);
         s << "commit " << branchRef << endl;
-        s << "mark :" << revnum << endl;
         s << "committer " << QString::fromUtf8(author) << ' ' << datetime << " -0000" << endl;
 
         Branch &br = repository->branches[branch];
@@ -226,18 +280,15 @@ void Repository::Transaction::commit()
             repository->fastImport.write("D " + df.toUtf8() + "\n");
 
     // write the file modifications
-    QHash<QString, FileProperties>::ConstIterator it = modifiedFiles.constBegin();
-    for ( ; it != modifiedFiles.constEnd(); ++it) {
-        repository->fastImport.write("M ", 2);
-        repository->fastImport.write(QByteArray::number(it->mode, 8));
-        repository->fastImport.write(" :", 2);
-        repository->fastImport.write(QByteArray::number(it->mark));
-        repository->fastImport.write(" ", 1);
-        repository->fastImport.write(it.key().toUtf8());
-        repository->fastImport.write("\n", 1);
-    }
-
-    repository->fastImport.write("\n");
+    repository->fastImport.write(modifiedFiles);
+
+    repository->fastImport.write("\nprogress Commit #" +
+                                 QByteArray::number(repository->commitCount) +
+                                 " branch " + branch +
+                                 " = SVN r" + QByteArray::number(revnum) + "\n\n");
+    printf(" %d modifications to \"%s\"",
+           deletedFiles.count() + modifiedFiles.count(),
+           qPrintable(repository->name));
 
     while (repository->fastImport.bytesToWrite())
         if (!repository->fastImport.waitForBytesWritten(-1))