Add a process cache to keep the number of processes under 100

[svn-all-fast-export.git] / src / repository.cpp

diff --git a/src/repository.cpp b/src/repository.cpp
index 29235e3650082a1a9a35c68ec5e074ca6f29e51d..e74ab207f4304ba592a2d476c5863a6b505bc1d2 100644 (file)
--- a/src/repository.cpp
+++ b/src/repository.cpp
@@ -18,6 +18,31 @@
 #include "repository.h"
 #include <QTextStream>
 #include <QDebug>
+#include <QLinkedList>
+
+static const int maxSimultaneousProcesses = 100;
+
+class ProcessCache: QLinkedList<Repository *>
+{
+public:
+    void touch(Repository *repo)
+    {
+        remove(repo);
+
+        // if the cache is too big, remove from the front
+        while (size() >= maxSimultaneousProcesses)
+            takeFirst()->closeFastImport();
+
+        // append to the end
+        append(repo);
+    }
+
+    inline void remove(Repository *repo)
+    {
+        removeOne(repo);
+    }
+};
+static ProcessCache processCache;
 
 Repository::Repository(const Rules::Repository &rule)
     : name(rule.name), commitCount(0), processHasStarted(false)
@@ -36,6 +61,11 @@ Repository::Repository(const Rules::Repository &rule)
 }
 
 Repository::~Repository()
+{
+    closeFastImport();
+}
+
+void Repository::closeFastImport()
 {
     if (fastImport.state() != QProcess::NotRunning) {
         fastImport.write("checkpoint\n");
@@ -47,6 +77,8 @@ Repository::~Repository()
                 qWarning() << "git-fast-import for repository" << name << "did not die";
         }
     }
+    processHasStarted = false;
+    processCache.remove(this);
 }
 
 void Repository::reloadBranches()
@@ -54,17 +86,17 @@ void Repository::reloadBranches()
     QProcess revParse;
     revParse.setWorkingDirectory(name);
     revParse.start("git", QStringList() << "rev-parse" << "--symbolic" << "--branches");
-    revParse.waitForFinished();
+    revParse.waitForFinished(-1);
 
     if (revParse.exitCode() == 0 && revParse.bytesAvailable()) {
-        startFastImport();
-
         while (revParse.canReadLine()) {
             QByteArray branchName = revParse.readLine().trimmed();
 
+            //qDebug() << "Repo" << name << "reloaded branch" << branchName;
             branches[branchName].created = 1;
             fastImport.write("reset refs/heads/" + branchName +
-                             "\nfrom refs/heads/" + branchName + "^0\n\n");
+                             "\nfrom refs/heads/" + branchName + "^0\n\n"
+                             "progress Branch refs/heads/" + branchName + " reloaded\n");
         }
     }
 }
@@ -72,12 +104,12 @@ void Repository::reloadBranches()
 void Repository::createBranch(const QString &branch, int revnum,
                               const QString &branchFrom, int)
 {
+    startFastImport();
     if (!branches.contains(branch)) {
         qWarning() << branch << "is not a known branch in repository" << name << endl
                    << "Going to create it automatically";
     }
 
-    startFastImport();
     QByteArray branchRef = branch.toUtf8();
     if (!branchRef.startsWith("refs/"))
         branchRef.prepend("refs/heads/");
@@ -103,12 +135,14 @@ void Repository::createBranch(const QString &branch, int revnum,
         exit(1);
     }
 
-    fastImport.write("reset " + branchRef + "\nfrom " + branchFromRef + "\n\n");
+    fastImport.write("reset " + branchRef + "\nfrom " + branchFromRef + "\n\n"
+        "progress Branch " + branchRef + " created from " + branchFromRef + "\n\n");
 }
 
 Repository::Transaction *Repository::newTransaction(const QString &branch, const QString &svnprefix,
                                                     int revnum)
 {
+    startFastImport();
     if (!branches.contains(branch)) {
         qWarning() << branch << "is not a known branch in repository" << name << endl
                    << "Going to create it automatically";
@@ -122,7 +156,6 @@ Repository::Transaction *Repository::newTransaction(const QString &branch, const
     txn->revnum = revnum;
     txn->lastmark = revnum;
 
-    startFastImport();
     if ((++commitCount % 10000) == 0)
         // write everything to disk every 10000 commits
         fastImport.write("checkpoint\n");
@@ -144,10 +177,12 @@ void Repository::startFastImport()
         fastImport.setProcessChannelMode(QProcess::MergedChannels);
 
 #ifndef DRY_RUN
-        fastImport.start("git-fast-import", QStringList());
+        fastImport.start("git", QStringList() << "fast-import");
 #else
         fastImport.start("/bin/cat", QStringList());
 #endif
+
+        reloadBranches();
     }
 }
 
@@ -195,6 +230,8 @@ QIODevice *Repository::Transaction::addFile(const QString &path, int mode, qint6
 
 void Repository::Transaction::commit()
 {
+    processCache.touch(repository);
+
     // create the commit message
     QByteArray message = log;
     if (!message.endsWith('\n'))
@@ -243,7 +280,13 @@ void Repository::Transaction::commit()
         repository->fastImport.write("\n", 1);
     }
 
-    repository->fastImport.write("\n");
+    repository->fastImport.write("\nprogress Commit #" +
+                                 QByteArray::number(repository->commitCount) +
+                                 " branch " + branch +
+                                 " = SVN r" + QByteArray::number(revnum) + "\n\n");
+    printf(" %d modifications to \"%s\"",
+           deletedFiles.count() + modifiedFiles.count(),
+           qPrintable(repository->name));
 
     while (repository->fastImport.bytesToWrite())
         if (!repository->fastImport.waitForBytesWritten(-1))