From boxbackup-dev@boxbackup.org  Mon Jun 22 06:33:00 2009
From: boxbackup-dev@boxbackup.org (boxbackup-dev@boxbackup.org)
Date: Mon, 22 Jun 2009 06:33:00 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2528 - box/trunk/bin/bbackupd
Message-ID: <20090622053300.165EC325037@www.boxbackup.org>

Author: chris
Date: 2009-06-22 06:32:59 +0100 (Mon, 22 Jun 2009)
New Revision: 2528

Modified:
   box/trunk/bin/bbackupd/BackupClientDirectoryRecord.cpp
Log:
Ignore stat() errors on excluded directories.


Modified: box/trunk/bin/bbackupd/BackupClientDirectoryRecord.cpp
===================================================================
--- box/trunk/bin/bbackupd/BackupClientDirectoryRecord.cpp	2009-05-22 20:01:51 UTC (rev 2527)
+++ box/trunk/bin/bbackupd/BackupClientDirectoryRecord.cpp	2009-06-22 05:32:59 UTC (rev 2528)
@@ -294,14 +294,21 @@
 				#else
 				if(EMU_LSTAT(filename.c_str(), &file_st) != 0)
 				{
-					// Report the error (logs and 
-					// eventual email to administrator)
- 					rNotifier.NotifyFileStatFailed(this, 
- 						filename, strerror(errno));
+					if(!(rParams.mrContext.ExcludeDir(
+						filename)))
+					{
+						// Report the error (logs and 
+						// eventual email to
+						// administrator)
+ 						rNotifier.NotifyFileStatFailed(
+							this, filename,
+							strerror(errno));
 					
-					// FIXME move to NotifyFileStatFailed()
-					SetErrorWhenReadingFilesystemObject(
-						rParams, filename.c_str());
+						// FIXME move to
+						// NotifyFileStatFailed()
+						SetErrorWhenReadingFilesystemObject(
+							rParams, filename.c_str());
+					}
 
 					// Ignore this entry for now.
 					continue;


From boxbackup-dev@boxbackup.org  Sat Jun 27 12:28:16 2009
From: boxbackup-dev@boxbackup.org (boxbackup-dev@boxbackup.org)
Date: Sat, 27 Jun 2009 12:28:16 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2529 - box/trunk/lib/raidfile
Message-ID: <20090627112816.68970325020@www.boxbackup.org>

Author: chris
Date: 2009-06-27 12:28:15 +0100 (Sat, 27 Jun 2009)
New Revision: 2529

Modified:
   box/trunk/lib/raidfile/RaidFileWrite.cpp
Log:
Improve RaidFileWrite error logging.


Modified: box/trunk/lib/raidfile/RaidFileWrite.cpp
===================================================================
--- box/trunk/lib/raidfile/RaidFileWrite.cpp	2009-06-22 05:32:59 UTC (rev 2528)
+++ box/trunk/lib/raidfile/RaidFileWrite.cpp	2009-06-27 11:28:15 UTC (rev 2529)
@@ -113,6 +113,7 @@
 		S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH);
 	if(mOSFileHandle == -1)
 	{
+		BOX_LOG_SYS_ERROR("Failed to open file: " << writeFilename);
 		THROW_EXCEPTION(RaidFileException, ErrorOpeningWriteFile)
 	}
 	
@@ -274,13 +275,15 @@
 	if(::unlink(renameTo.c_str()) != 0 && 
 		GetLastError() != ERROR_FILE_NOT_FOUND)
 	{
-		BOX_LOG_WIN_ERROR("failed to delete file: " << renameTo);
+		BOX_LOG_WIN_ERROR("Failed to delete file: " << renameTo);
 		THROW_EXCEPTION(RaidFileException, OSError)
 	}
 #endif
 
 	if(::rename(renameFrom.c_str(), renameTo.c_str()) != 0)
 	{
+		BOX_LOG_SYS_ERROR("Failed to rename file: " << renameFrom <<
+			" to " << renameTo);
 		THROW_EXCEPTION(RaidFileException, OSError)
 	}
 	
@@ -335,6 +338,7 @@
 		::close(mOSFileHandle) != 0)
 #endif // !WIN32
 	{
+		BOX_LOG_SYS_ERROR("Failed to delete file: " << writeFilename);
 		THROW_EXCEPTION(RaidFileException, OSError)
 	}
 	
@@ -562,6 +566,8 @@
 			ASSERT((::lseek(parity, 0, SEEK_CUR) % blockSize) == 0);
 			if(::write(parity, &sw, sizeof(sw)) != sizeof(sw))
 			{
+				BOX_LOG_SYS_ERROR("Failed to write to file: " <<
+					writeFilename);
 				THROW_EXCEPTION(RaidFileException, OSError)
 			}
 		}
@@ -600,6 +606,8 @@
 		// Finally delete the write file
 		if(::unlink(writeFilename.c_str()) != 0)
 		{
+			BOX_LOG_SYS_ERROR("Failed to delete file: " <<
+				writeFilename);
 			THROW_EXCEPTION(RaidFileException, OSError)
 		}
 	}


From boxbackup-dev@boxbackup.org  Sat Jun 27 12:34:20 2009
From: boxbackup-dev@boxbackup.org (boxbackup-dev@boxbackup.org)
Date: Sat, 27 Jun 2009 12:34:20 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2530 - box/trunk/lib/backupstore
Message-ID: <20090627113420.C61C1325020@www.boxbackup.org>

Author: chris
Date: 2009-06-27 12:34:19 +0100 (Sat, 27 Jun 2009)
New Revision: 2530

Modified:
   box/trunk/lib/backupstore/BackupStoreAccountDatabase.cpp
   box/trunk/lib/backupstore/BackupStoreAccountDatabase.h
Log:
Make BackupStoreAccountDatabase::AddEntry and 
BackupStoreAccountDatabase::GetEntry return a copy of the new entry. 


Modified: box/trunk/lib/backupstore/BackupStoreAccountDatabase.cpp
===================================================================
--- box/trunk/lib/backupstore/BackupStoreAccountDatabase.cpp	2009-06-27 11:28:15 UTC (rev 2529)
+++ box/trunk/lib/backupstore/BackupStoreAccountDatabase.cpp	2009-06-27 11:34:19 UTC (rev 2530)
@@ -289,7 +289,8 @@
 //		Created: 2003/08/21
 //
 // --------------------------------------------------------------------------
-const BackupStoreAccountDatabase::Entry &BackupStoreAccountDatabase::GetEntry(int32_t ID) const
+BackupStoreAccountDatabase::Entry BackupStoreAccountDatabase::GetEntry(
+	int32_t ID) const
 {
 	// Check that we're using the latest version of the database
 	CheckUpToDate();
@@ -311,12 +312,14 @@
 //		Created: 2003/08/21
 //
 // --------------------------------------------------------------------------
-void BackupStoreAccountDatabase::AddEntry(int32_t ID, int DiscSet)
+BackupStoreAccountDatabase::Entry BackupStoreAccountDatabase::AddEntry(
+	int32_t ID, int DiscSet)
 {
 	// Check that we're using the latest version of the database
 	CheckUpToDate();
 
 	pImpl->mDatabase[ID] = Entry(ID, DiscSet);
+	return pImpl->mDatabase[ID];
 }
 
 

Modified: box/trunk/lib/backupstore/BackupStoreAccountDatabase.h
===================================================================
--- box/trunk/lib/backupstore/BackupStoreAccountDatabase.h	2009-06-27 11:28:15 UTC (rev 2529)
+++ box/trunk/lib/backupstore/BackupStoreAccountDatabase.h	2009-06-27 11:34:19 UTC (rev 2530)
@@ -55,8 +55,8 @@
 	};
 
 	bool EntryExists(int32_t ID) const;
-	const Entry &GetEntry(int32_t ID) const;
-	void AddEntry(int32_t ID, int DiscSet);
+	Entry GetEntry(int32_t ID) const;
+	Entry AddEntry(int32_t ID, int DiscSet);
 	void DeleteEntry(int32_t ID);
 
 	// This interface should change in the future. But for now it'll do.


From boxbackup-dev@boxbackup.org  Sat Jun 27 12:34:41 2009
From: boxbackup-dev@boxbackup.org (boxbackup-dev@boxbackup.org)
Date: Sat, 27 Jun 2009 12:34:41 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2531 - box/trunk/lib/backupclient
Message-ID: <20090627113441.D8567325020@www.boxbackup.org>

Author: chris
Date: 2009-06-27 12:34:41 +0100 (Sat, 27 Jun 2009)
New Revision: 2531

Modified:
   box/trunk/lib/backupclient/BackupStoreException.txt
Log:
Add a new exception type for refcount database errors.


Modified: box/trunk/lib/backupclient/BackupStoreException.txt
===================================================================
--- box/trunk/lib/backupclient/BackupStoreException.txt	2009-06-27 11:34:19 UTC (rev 2530)
+++ box/trunk/lib/backupclient/BackupStoreException.txt	2009-06-27 11:34:41 UTC (rev 2531)
@@ -68,3 +68,4 @@
 IncompatibleFromAndDiffFiles	65	Attempt to use a diff and a from file together, when they're not related
 DiffFromIDNotFoundInDirectory	66	When uploading via a diff, the diff from file must be in the same directory
 PatchChainInfoBadInDirectory	67	A directory contains inconsistent information. Run bbstoreaccounts check to fix it.
+UnknownObjectRefCountRequested	68	A reference count was requested for an object whose reference count is not known.


From boxbackup-dev@boxbackup.org  Sat Jun 27 12:35:02 2009
From: boxbackup-dev@boxbackup.org (boxbackup-dev@boxbackup.org)
Date: Sat, 27 Jun 2009 12:35:02 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2532 - box/trunk/lib/common
Message-ID: <20090627113502.ACD65326026@www.boxbackup.org>

Author: chris
Date: 2009-06-27 12:35:02 +0100 (Sat, 27 Jun 2009)
New Revision: 2532

Modified:
   box/trunk/lib/common/Utils.cpp
Log:
Make FileExists take a std::string instead of a const char * for C++ 
style.


Modified: box/trunk/lib/common/Utils.cpp
===================================================================
--- box/trunk/lib/common/Utils.cpp	2009-06-27 11:34:41 UTC (rev 2531)
+++ box/trunk/lib/common/Utils.cpp	2009-06-27 11:35:02 UTC (rev 2532)
@@ -156,15 +156,16 @@
 // --------------------------------------------------------------------------
 //
 // Function
-//		Name:    FileExists(const char *)
+//		Name:    FileExists(const std::string& rFilename)
 //		Purpose: Does a file exist?
 //		Created: 20/11/03
 //
 // --------------------------------------------------------------------------
-bool FileExists(const char *Filename, int64_t *pFileSize, bool TreatLinksAsNotExisting)
+bool FileExists(const std::string& rFilename, int64_t *pFileSize,
+	bool TreatLinksAsNotExisting)
 {
 	EMU_STRUCT_STAT st;
-	if(EMU_LSTAT(Filename, &st) != 0)
+	if(EMU_LSTAT(rFilename.c_str(), &st) != 0)
 	{
 		if(errno == ENOENT)
 		{


From boxbackup-dev@boxbackup.org  Sat Jun 27 12:35:15 2009
From: boxbackup-dev@boxbackup.org (boxbackup-dev@boxbackup.org)
Date: Sat, 27 Jun 2009 12:35:15 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2533 - box/trunk/lib/common
Message-ID: <20090627113515.ACC32325020@www.boxbackup.org>

Author: chris
Date: 2009-06-27 12:35:15 +0100 (Sat, 27 Jun 2009)
New Revision: 2533

Modified:
   box/trunk/lib/common/Utils.h
Log:
Second part of FileExists change.


Modified: box/trunk/lib/common/Utils.h
===================================================================
--- box/trunk/lib/common/Utils.h	2009-06-27 11:35:02 UTC (rev 2532)
+++ box/trunk/lib/common/Utils.h	2009-06-27 11:35:15 UTC (rev 2533)
@@ -21,7 +21,8 @@
 	void DumpStackBacktrace();
 #endif
 
-bool FileExists(const char *Filename, int64_t *pFileSize = 0, bool TreatLinksAsNotExisting = false);
+bool FileExists(const std::string& rFilename, int64_t *pFileSize = 0,
+	bool TreatLinksAsNotExisting = false);
 
 enum
 {


From boxbackup-dev@boxbackup.org  Sat Jun 27 12:38:53 2009
From: boxbackup-dev@boxbackup.org (boxbackup-dev@boxbackup.org)
Date: Sat, 27 Jun 2009 12:38:53 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2534 - box/trunk/lib/backupstore
Message-ID: <20090627113853.12D29325020@www.boxbackup.org>

Author: chris
Date: 2009-06-27 12:38:52 +0100 (Sat, 27 Jun 2009)
New Revision: 2534

Added:
   box/trunk/lib/backupstore/BackupStoreRefCountDatabase.cpp
   box/trunk/lib/backupstore/BackupStoreRefCountDatabase.h
Modified:
   box/trunk/lib/backupstore/BackupStoreAccounts.cpp
   box/trunk/lib/backupstore/BackupStoreAccounts.h
Log:
Add code for BackupStoreRefCountDatabase.

Create a refcount database when creating a new account.

Add an easier way to get the account root directory.


Modified: box/trunk/lib/backupstore/BackupStoreAccounts.cpp
===================================================================
--- box/trunk/lib/backupstore/BackupStoreAccounts.cpp	2009-06-27 11:35:15 UTC (rev 2533)
+++ box/trunk/lib/backupstore/BackupStoreAccounts.cpp	2009-06-27 11:38:52 UTC (rev 2534)
@@ -14,6 +14,7 @@
 #include "BoxPortsAndFiles.h"
 #include "BackupStoreAccounts.h"
 #include "BackupStoreAccountDatabase.h"
+#include "BackupStoreRefCountDatabase.h"
 #include "RaidFileWrite.h"
 #include "BackupStoreInfo.h"
 #include "BackupStoreDirectory.h"
@@ -61,6 +62,10 @@
 // --------------------------------------------------------------------------
 void BackupStoreAccounts::Create(int32_t ID, int DiscSet, int64_t SizeSoftLimit, int64_t SizeHardLimit, const std::string &rAsUsername)
 {
+	// Create the entry in the database
+	BackupStoreAccountDatabase::Entry Entry(mrDatabase.AddEntry(ID,
+		DiscSet));
+	
 	{
 		// Become the user specified in the config file?
 		std::auto_ptr<UnixUser> user;
@@ -100,15 +105,17 @@
 		
 		// Save it back
 		info->Save();
+
+		// Create the refcount database
+		BackupStoreRefCountDatabase::CreateNew(Entry);
+		std::auto_ptr<BackupStoreRefCountDatabase> refcount(
+			BackupStoreRefCountDatabase::Load(Entry, false));
+		refcount->AddReference(BACKUPSTORE_ROOT_DIRECTORY_ID);
 	}
 
 	// As the original user...
-
-	// Create the entry in the database
-	mrDatabase.AddEntry(ID, DiscSet);
-	
 	// Write the database back
-	mrDatabase.Write();	
+	mrDatabase.Write();
 }
 
 
@@ -138,7 +145,7 @@
 //		Created: 2003/08/21
 //
 // --------------------------------------------------------------------------
-std::string BackupStoreAccounts::MakeAccountRootDir(int32_t ID, int DiscSet) const
+std::string BackupStoreAccounts::MakeAccountRootDir(int32_t ID, int DiscSet)
 {
 	char accid[64];	// big enough!
 	::sprintf(accid, "%08x" DIRECTORY_SEPARATOR, ID);

Modified: box/trunk/lib/backupstore/BackupStoreAccounts.h
===================================================================
--- box/trunk/lib/backupstore/BackupStoreAccounts.h	2009-06-27 11:35:15 UTC (rev 2533)
+++ box/trunk/lib/backupstore/BackupStoreAccounts.h	2009-06-27 11:38:52 UTC (rev 2534)
@@ -12,7 +12,7 @@
 
 #include <string>
 
-class BackupStoreAccountDatabase;
+#include "BackupStoreAccountDatabase.h"
 
 // --------------------------------------------------------------------------
 //
@@ -35,9 +35,14 @@
 
 	bool AccountExists(int32_t ID);
 	void GetAccountRoot(int32_t ID, std::string &rRootDirOut, int &rDiscSetOut) const;
+	static std::string GetAccountRoot(const
+		BackupStoreAccountDatabase::Entry &rEntry)
+	{
+		return MakeAccountRootDir(rEntry.GetID(), rEntry.GetDiscSet());
+	}
 
 private:
-	std::string MakeAccountRootDir(int32_t ID, int DiscSet) const;
+	static std::string MakeAccountRootDir(int32_t ID, int DiscSet);
 
 private:
 	BackupStoreAccountDatabase &mrDatabase;

Copied: box/trunk/lib/backupstore/BackupStoreRefCountDatabase.cpp (from rev 2527, box/trunk/lib/backupstore/BackupStoreInfo.cpp)
===================================================================
--- box/trunk/lib/backupstore/BackupStoreRefCountDatabase.cpp	                        (rev 0)
+++ box/trunk/lib/backupstore/BackupStoreRefCountDatabase.cpp	2009-06-27 11:38:52 UTC (rev 2534)
@@ -0,0 +1,319 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreRefCountDatabase.cpp
+//		Purpose: Backup store object reference count database storage
+//		Created: 2009/06/01
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <algorithm>
+
+#include "BackupStoreRefCountDatabase.h"
+#include "BackupStoreException.h"
+#include "BackupStoreAccountDatabase.h"
+#include "BackupStoreAccounts.h"
+#include "RaidFileController.h"
+#include "RaidFileUtil.h"
+#include "RaidFileException.h"
+
+#include "MemLeakFindOn.h"
+
+#define REFCOUNT_MAGIC_VALUE	0x52656643 // RefC
+#define REFCOUNT_FILENAME	"refcount"
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreRefCountDatabase::BackupStoreRefCountDatabase()
+//		Purpose: Default constructor
+//		Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+BackupStoreRefCountDatabase::BackupStoreRefCountDatabase(const
+	BackupStoreAccountDatabase::Entry& rAccount)
+: mAccount(rAccount),
+  mFilename(GetFilename(rAccount)),
+  mReadOnly(true),
+  mIsModified(false)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreRefCountDatabase::~BackupStoreRefCountDatabase
+//		Purpose: Destructor
+//		Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+BackupStoreRefCountDatabase::~BackupStoreRefCountDatabase()
+{
+}
+
+std::string BackupStoreRefCountDatabase::GetFilename(const
+	BackupStoreAccountDatabase::Entry& rAccount)
+{
+	std::string RootDir = BackupStoreAccounts::GetAccountRoot(rAccount);
+	ASSERT(RootDir[RootDir.size() - 1] == '/' ||
+		RootDir[RootDir.size() - 1] == DIRECTORY_SEPARATOR_ASCHAR);
+
+	std::string fn(RootDir + "refcount.db");
+	RaidFileController &rcontroller(RaidFileController::GetController());
+	RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(rAccount.GetDiscSet()));
+	return RaidFileUtil::MakeWriteFileName(rdiscSet, fn);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreRefCountDatabase::Create(int32_t,
+//			 const std::string &, int, bool)
+//		Purpose: Create a new database, overwriting an existing
+//			 one only if AllowOverwrite is true.
+//		Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+void BackupStoreRefCountDatabase::Create(const
+	BackupStoreAccountDatabase::Entry& rAccount, bool AllowOverwrite)
+{
+	// Initial header
+	refcount_StreamFormat hdr;
+	hdr.mMagicValue = htonl(REFCOUNT_MAGIC_VALUE);
+	hdr.mAccountID = htonl(rAccount.GetID());
+	
+	// Generate the filename
+	std::string Filename = GetFilename(rAccount);
+
+	// Open the file for writing
+	if (FileExists(Filename) && !AllowOverwrite)
+	{
+		BOX_ERROR("Attempted to overwrite refcount database file: " <<
+			Filename);
+		THROW_EXCEPTION(RaidFileException, CannotOverwriteExistingFile);
+	}
+
+	int flags = O_CREAT | O_BINARY | O_RDWR;
+	if (!AllowOverwrite)
+	{
+		flags |= O_EXCL;
+	}
+
+	std::auto_ptr<FileStream> DatabaseFile(new FileStream(Filename, flags));
+	
+	// Write header
+	DatabaseFile->Write(&hdr, sizeof(hdr));
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreRefCountDatabase::Load(int32_t AccountID,
+//			 BackupStoreAccountDatabase& rAccountDatabase,
+//			 bool ReadOnly);
+//		Purpose: Loads the info from disc, given the root
+//			 information. Can be marked as read only.
+//		Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<BackupStoreRefCountDatabase> BackupStoreRefCountDatabase::Load(
+	const BackupStoreAccountDatabase::Entry& rAccount, bool ReadOnly)
+{
+	// Generate the filename
+	std::string filename = GetFilename(rAccount);
+	int flags = ReadOnly ? O_RDONLY : O_RDWR;
+
+	// Open the file for read/write
+	std::auto_ptr<FileStream> dbfile(new FileStream(filename,
+		flags | O_BINARY));
+	
+	// Read in a header
+	refcount_StreamFormat hdr;
+	if(!dbfile->ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */))
+	{
+		THROW_EXCEPTION(BackupStoreException, CouldNotLoadStoreInfo)
+	}
+	
+	// Check it
+	if(ntohl(hdr.mMagicValue) != REFCOUNT_MAGIC_VALUE ||
+		(int32_t)ntohl(hdr.mAccountID) != rAccount.GetID())
+	{
+		THROW_EXCEPTION(BackupStoreException, BadStoreInfoOnLoad)
+	}
+	
+	// Make new object
+	std::auto_ptr<BackupStoreRefCountDatabase> refcount(new BackupStoreRefCountDatabase(rAccount));
+	
+	// Put in basic location info
+	refcount->mReadOnly = ReadOnly;
+	refcount->mapDatabaseFile = dbfile;
+	
+	// return it to caller
+	return refcount;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreRefCountDatabase::Save()
+//		Purpose: Save modified info back to disc
+//		Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+/*
+void BackupStoreRefCountDatabase::Save()
+{
+	// Make sure we're initialised (although should never come to this)
+	if(mFilename.empty() || mAccount.GetID() == 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, Internal)
+	}
+
+	// Can we do this?
+	if(mReadOnly)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+	}
+	
+	// Then... open a write file
+	RaidFileWrite rf(mAccount.GetDiscSet(), mFilename);
+	rf.Open(true);		// allow overwriting
+	
+	// Make header
+	info_StreamFormat hdr;
+	hdr.mMagicValue 			= htonl(INFO_MAGIC_VALUE);
+	hdr.mAccountID 				= htonl(mAccountID);
+	hdr.mClientStoreMarker			= box_hton64(mClientStoreMarker);
+	hdr.mLastObjectIDUsed			= box_hton64(mLastObjectIDUsed);
+	hdr.mBlocksUsed 				= box_hton64(mBlocksUsed);
+	hdr.mBlocksInOldFiles 			= box_hton64(mBlocksInOldFiles);
+	hdr.mBlocksInDeletedFiles 		= box_hton64(mBlocksInDeletedFiles);
+	hdr.mBlocksInDirectories		= box_hton64(mBlocksInDirectories);
+	hdr.mBlocksSoftLimit			= box_hton64(mBlocksSoftLimit);
+	hdr.mBlocksHardLimit			= box_hton64(mBlocksHardLimit);
+	hdr.mCurrentMarkNumber			= 0;
+	hdr.mOptionsPresent				= 0;
+	hdr.mNumberDeletedDirectories	= box_hton64(mDeletedDirectories.size());
+	
+	// Write header
+	rf.Write(&hdr, sizeof(hdr));
+	
+	// Write the deleted object list
+	if(mDeletedDirectories.size() > 0)
+	{
+		int64_t objs[NUM_DELETED_DIRS_BLOCK];
+		
+		int tosave = mDeletedDirectories.size();
+		std::vector<int64_t>::iterator i(mDeletedDirectories.begin());
+		while(tosave > 0)
+		{
+			// How many in this one?
+			int b = (tosave > NUM_DELETED_DIRS_BLOCK)?NUM_DELETED_DIRS_BLOCK:((int)(tosave));
+			
+			// Add them
+			for(int t = 0; t < b; ++t)
+			{
+				ASSERT(i != mDeletedDirectories.end());
+				objs[t] = box_hton64((*i));
+				i++;
+			}
+
+			// Write			
+			rf.Write(objs, b * sizeof(int64_t));
+			
+			// Number saved
+			tosave -= b;
+		}
+	}
+
+	// Commit it to disc, converting it to RAID now
+	rf.Commit(true);
+	
+	// Mark is as not modified
+	mIsModified = false;
+}
+*/
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreRefCountDatabase::GetRefCount(int64_t
+//			 ObjectID)
+//		Purpose: Get the number of references to the specified object
+//			 out of the database
+//		Created: 2009/06/01
+//
+// --------------------------------------------------------------------------
+int32_t BackupStoreRefCountDatabase::GetRefCount(int64_t ObjectID) const
+{
+	IOStream::pos_type offset = GetOffset(ObjectID);
+
+	if (GetSize() < offset + GetEntrySize())
+	{
+		BOX_ERROR("attempted read of unknown refcount for object " <<
+			BOX_FORMAT_OBJECTID(ObjectID));
+		THROW_EXCEPTION(BackupStoreException,
+			UnknownObjectRefCountRequested);
+	}
+
+	mapDatabaseFile->Seek(offset, SEEK_SET);
+
+	refcount_t refcount;
+	if (mapDatabaseFile->Read(&refcount, sizeof(refcount)) !=
+		sizeof(refcount))
+	{
+		BOX_LOG_SYS_ERROR("short read on refcount database: " <<
+			mFilename);
+		THROW_EXCEPTION(BackupStoreException, CouldNotLoadStoreInfo);
+	}
+
+	return ntohl(refcount);
+}
+
+int64_t BackupStoreRefCountDatabase::GetLastObjectIDUsed() const
+{
+	return (GetSize() - sizeof(refcount_StreamFormat)) /
+		sizeof(refcount_t);
+}
+
+void BackupStoreRefCountDatabase::AddReference(int64_t ObjectID)
+{
+	refcount_t refcount;
+
+	if (ObjectID > GetLastObjectIDUsed())
+	{
+		// new object, assume no previous references
+		refcount = 0;
+	}
+	else
+	{
+		// read previous value from database
+		refcount = GetRefCount(ObjectID);
+	}
+
+	refcount++;
+
+	SetRefCount(ObjectID, refcount);
+}
+
+void BackupStoreRefCountDatabase::SetRefCount(int64_t ObjectID,
+	refcount_t NewRefCount)
+{
+	IOStream::pos_type offset = GetOffset(ObjectID);
+	mapDatabaseFile->Seek(offset, SEEK_SET);
+	refcount_t RefCountNetOrder = htonl(NewRefCount);
+	mapDatabaseFile->Write(&RefCountNetOrder, sizeof(RefCountNetOrder));
+}
+
+bool BackupStoreRefCountDatabase::RemoveReference(int64_t ObjectID)
+{
+	refcount_t refcount = GetRefCount(ObjectID); // must exist in database
+	ASSERT(refcount > 0);
+	refcount--;
+	SetRefCount(ObjectID, refcount);
+	return (refcount > 0);
+}
+

Copied: box/trunk/lib/backupstore/BackupStoreRefCountDatabase.h (from rev 2527, box/trunk/lib/backupstore/BackupStoreInfo.h)
===================================================================
--- box/trunk/lib/backupstore/BackupStoreRefCountDatabase.h	                        (rev 0)
+++ box/trunk/lib/backupstore/BackupStoreRefCountDatabase.h	2009-06-27 11:38:52 UTC (rev 2534)
@@ -0,0 +1,125 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreRefCountDatabase.h
+//		Purpose: Main backup store information storage
+//		Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREREFCOUNTDATABASE__H
+#define BACKUPSTOREREFCOUNTDATABASE__H
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "BackupStoreAccountDatabase.h"
+#include "FileStream.h"
+
+class BackupStoreCheck;
+class BackupStoreContext;
+
+// set packing to one byte
+#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS
+#include "BeginStructPackForWire.h"
+#else
+BEGIN_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+typedef struct
+{
+	uint32_t mMagicValue;	// also the version number
+	uint32_t mAccountID;
+} refcount_StreamFormat;
+
+// Use default packing
+#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS
+#include "EndStructPackForWire.h"
+#else
+END_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    BackupStoreRefCountDatabase
+//		Purpose: Backup store reference count database storage
+//		Created: 2009/06/01
+//
+// --------------------------------------------------------------------------
+class BackupStoreRefCountDatabase
+{
+	friend class BackupStoreCheck;
+	friend class BackupStoreContext;
+public:
+	~BackupStoreRefCountDatabase();
+private:
+	// Creation through static functions only
+	BackupStoreRefCountDatabase(const BackupStoreAccountDatabase::Entry&
+		rAccount);
+	// No copying allowed
+	BackupStoreRefCountDatabase(const BackupStoreRefCountDatabase &);
+	
+public:
+	// Create a new database for a new account. This method will refuse
+	// to overwrite any existing file.
+	static void CreateNew(const BackupStoreAccountDatabase::Entry& rAccount)
+	{
+		Create(rAccount, false);
+	}
+	
+	// Load it from the store
+	static std::auto_ptr<BackupStoreRefCountDatabase> Load(const
+		BackupStoreAccountDatabase::Entry& rAccount, bool ReadOnly);
+	
+	// Data access functions
+	int32_t GetRefCount(int64_t ObjectID) const;
+	int64_t GetLastObjectIDUsed() const;
+	
+	// Data modification functions
+	void AddReference(int64_t ObjectID);
+	bool RemoveReference(int64_t ObjectID);
+
+private:
+	// Create a new database for an existing account. Used during
+	// account checking if opening the old database throws an exception.
+	// This method will overwrite any existing file.
+	static void CreateForRegeneration(const
+		BackupStoreAccountDatabase::Entry& rAccount)
+	{
+		Create(rAccount, true);
+	}
+
+	static void Create(const BackupStoreAccountDatabase::Entry& rAccount,
+		bool AllowOverwrite);
+
+	typedef int32_t refcount_t;
+
+	static std::string GetFilename(const BackupStoreAccountDatabase::Entry&
+		rAccount);
+	IOStream::pos_type GetSize() const
+	{
+		return mapDatabaseFile->GetPosition() +
+			mapDatabaseFile->BytesLeftToRead();
+	}
+	IOStream::pos_type GetEntrySize() const
+	{
+		return sizeof(refcount_t);
+	}
+	IOStream::pos_type GetOffset(int64_t ObjectID) const
+	{
+		return ((ObjectID - 1) * GetEntrySize()) +
+			sizeof(refcount_StreamFormat);
+	}
+	void SetRefCount(int64_t ObjectID, refcount_t NewRefCount);
+	
+	// Location information
+	BackupStoreAccountDatabase::Entry mAccount;
+	std::string mFilename;
+	bool mReadOnly;
+	bool mIsModified;
+	std::auto_ptr<FileStream> mapDatabaseFile;
+};
+
+#endif // BACKUPSTOREREFCOUNTDATABASE__H


From boxbackup-dev@boxbackup.org  Sun Jun 28 13:04:26 2009
From: boxbackup-dev@boxbackup.org (boxbackup-dev@boxbackup.org)
Date: Sun, 28 Jun 2009 13:04:26 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2535 - in box/trunk: bin/bbstored test/backupstore
Message-ID: <20090628120426.429F1325020@www.boxbackup.org>

Author: chris
Date: 2009-06-28 13:04:25 +0100 (Sun, 28 Jun 2009)
New Revision: 2535

Modified:
   box/trunk/bin/bbstored/BackupStoreContext.cpp
   box/trunk/bin/bbstored/BackupStoreContext.h
   box/trunk/test/backupstore/testbackupstore.cpp
Log:
Update reference counts when files and directories are uploaded.


Modified: box/trunk/bin/bbstored/BackupStoreContext.cpp
===================================================================
--- box/trunk/bin/bbstored/BackupStoreContext.cpp	2009-06-27 11:38:52 UTC (rev 2534)
+++ box/trunk/bin/bbstored/BackupStoreContext.cpp	2009-06-28 12:04:25 UTC (rev 2535)
@@ -191,6 +191,23 @@
 	
 	// Keep the pointer to it
 	mpStoreInfo = i;
+
+	BackupStoreAccountDatabase::Entry account(mClientID, mStoreDiscSet);
+
+	// try to load the reference count database
+	try
+	{
+		mapRefCount = BackupStoreRefCountDatabase::Load(account, false);
+	}
+	catch(BoxException &e)
+	{
+		BOX_WARNING("Reference count database is missing or corrupted, "
+			"creating a new one, expect housekeeping to find and "
+			"fix problems with reference counts later.");
+		
+		BackupStoreRefCountDatabase::CreateForRegeneration(account);
+		mapRefCount = BackupStoreRefCountDatabase::Load(account, false);
+	}
 }
 
 
@@ -395,15 +412,18 @@
 // --------------------------------------------------------------------------
 //
 // Function
-//		Name:    BackupStoreContext::AddFile(IOStream &, int64_t, int64_t, int64_t, const BackupStoreFilename &, bool)
-//		Purpose: Add a file to the store, from a given stream, into a specified directory.
-//				 Returns object ID of the new file.
+//		Name:    BackupStoreContext::AddFile(IOStream &, int64_t,
+//			 int64_t, int64_t, const BackupStoreFilename &, bool)
+//		Purpose: Add a file to the store, from a given stream, into
+//			 a specified directory. Returns object ID of the new
+//			 file.
 //		Created: 2003/09/03
 //
 // --------------------------------------------------------------------------
-int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, int64_t ModificationTime,
-		int64_t AttributesHash, int64_t DiffFromFileID, const BackupStoreFilename &rFilename,
-		bool MarkFileWithSameNameAsOldVersions)
+int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory,
+	int64_t ModificationTime, int64_t AttributesHash,
+	int64_t DiffFromFileID, const BackupStoreFilename &rFilename,
+	bool MarkFileWithSameNameAsOldVersions)
 {
 	if(mpStoreInfo.get() == 0)
 	{
@@ -670,6 +690,9 @@
 	mpStoreInfo->ChangeBlocksUsed(blocksUsed);
 	mpStoreInfo->ChangeBlocksInOldFiles(blocksInOldFiles);
 	
+	// Increment reference count on the new directory to one
+	mapRefCount->AddReference(id);
+	
 	// Save the store info -- can cope if this exceptions because infomation
 	// will be rebuilt by housekeeping, and ID allocation can recover.
 	SaveStoreInfo();
@@ -768,8 +791,9 @@
 // --------------------------------------------------------------------------
 //
 // Function
-//		Name:    BackupStoreContext::DeleteFile(const BackupStoreFilename &, int64_t, int64_t &)
-//		Purpose: Deletes a file, returning true if the file existed. Object ID returned too, set to zero if not found.
+//		Name:    BackupStoreContext::UndeleteFile(int64_t, int64_t)
+//		Purpose: Undeletes a file, if it exists, returning true if
+//			 the file existed.
 //		Created: 2003/10/21
 //
 // --------------------------------------------------------------------------
@@ -933,8 +957,10 @@
 // --------------------------------------------------------------------------
 //
 // Function
-//		Name:    BackupStoreContext::AddDirectory(int64_t, const BackupStoreFilename &, bool &)
-//		Purpose: Creates a directory (or just returns the ID of an existing one). rAlreadyExists set appropraitely.
+//		Name:    BackupStoreContext::AddDirectory(int64_t,
+//			 const BackupStoreFilename &, bool &)
+//		Purpose: Creates a directory (or just returns the ID of an
+//			 existing one). rAlreadyExists set appropraitely.
 //		Created: 2003/09/04
 //
 // --------------------------------------------------------------------------
@@ -974,7 +1000,7 @@
 	// Allocate the next ID
 	int64_t id = AllocateObjectID();
 
-	// Create a blank directory with the given attributes on disc
+	// Create an empty directory with the given attributes on disc
 	std::string fn;
 	MakeObjectFilename(id, fn, true /* make sure the directory it's in exists */);
 	{
@@ -998,11 +1024,14 @@
 		// Not added to cache, so don't set the size in the directory
 	}
 	
-	// Then add it into the directory
+	// Then add it into the parent directory
 	try
 	{
 		dir.AddEntry(rFilename, 0 /* modification time */, id, 0 /* blocks used */, BackupStoreDirectory::Entry::Flags_Dir, 0 /* attributes mod time */);
 		SaveDirectory(dir, InDirectory);
+
+		// Increment reference count on the new directory to one
+		mapRefCount->AddReference(id);
 	}
 	catch(...)
 	{
@@ -1016,7 +1045,7 @@
 		// Don't worry about the incremented number in the store info
 		throw;	
 	}
-	
+
 	// Save the store info (may be postponed)
 	SaveStoreInfo();
 

Modified: box/trunk/bin/bbstored/BackupStoreContext.h
===================================================================
--- box/trunk/bin/bbstored/BackupStoreContext.h	2009-06-27 11:38:52 UTC (rev 2534)
+++ box/trunk/bin/bbstored/BackupStoreContext.h	2009-06-28 12:04:25 UTC (rev 2535)
@@ -14,6 +14,7 @@
 #include <map>
 #include <memory>
 
+#include "BackupStoreRefCountDatabase.h"
 #include "NamedLock.h"
 #include "ProtocolObject.h"
 #include "Utils.h"
@@ -137,7 +138,6 @@
 	void DeleteDirectoryRecurse(int64_t ObjectID, int64_t &rBlocksDeletedOut, bool Undelete);
 	int64_t AllocateObjectID();
 
-private:
 	int32_t mClientID;
 	HousekeepingInterface &mrDaemon;
 	int mProtocolPhase;
@@ -150,6 +150,9 @@
 	
 	// Store info
 	std::auto_ptr<BackupStoreInfo> mpStoreInfo;
+
+	// Refcount database
+	std::auto_ptr<BackupStoreRefCountDatabase> mapRefCount;
 	
 	// Directory cache
 	std::map<int64_t, BackupStoreDirectory*> mDirectoryCache;

Modified: box/trunk/test/backupstore/testbackupstore.cpp
===================================================================
--- box/trunk/test/backupstore/testbackupstore.cpp	2009-06-27 11:38:52 UTC (rev 2534)
+++ box/trunk/test/backupstore/testbackupstore.cpp	2009-06-28 12:04:25 UTC (rev 2535)
@@ -34,6 +34,8 @@
 #include "BackupClientFileAttributes.h"
 #include "BackupClientCryptoKeys.h"
 #include "ServerControl.h"
+#include "BackupStoreAccountDatabase.h"
+#include "BackupStoreRefCountDatabase.h"
 
 #include "MemLeakFindOn.h"
 
@@ -470,8 +472,28 @@
 	TEST_THAT(dirs == 0 || dirs == 2);
 }
 
-int64_t create_test_data_subdirs(BackupProtocolClient &protocol, int64_t indir, const char *name, int depth)
+void create_file_in_dir(std::string name, std::string source, int64_t parentId,
+	BackupProtocolClient &protocol, BackupStoreRefCountDatabase& rRefCount)
 {
+	BackupStoreFilenameClear name_encoded("file_One");
+	std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile(
+		source.c_str(), parentId, name_encoded));
+	std::auto_ptr<BackupProtocolClientSuccess> stored(
+		protocol.QueryStoreFile(
+			parentId,
+			0x123456789abcdefLL,		/* modification time */
+			0x7362383249872dfLL,		/* attr hash */
+			0,				/* diff from ID */
+			name_encoded,
+			*upload));
+	int64_t objectId = stored->GetObjectID();
+	TEST_EQUAL(objectId, rRefCount.GetLastObjectIDUsed());
+	TEST_EQUAL(1, rRefCount.GetRefCount(objectId))
+}
+
+int64_t create_test_data_subdirs(BackupProtocolClient &protocol, int64_t indir,
+	const char *name, int depth, BackupStoreRefCountDatabase& rRefCount)
+{
 	// Create a directory
 	int64_t subdirid = 0;
 	BackupStoreFilenameClear dirname(name);
@@ -486,49 +508,31 @@
 	}
 	
 	printf("Create subdirs, depth = %d, dirid = %llx\n", depth, subdirid);
+
+	std::auto_ptr<BackupStoreAccountDatabase> apAccounts(
+		BackupStoreAccountDatabase::Read("testfiles/accounts.txt"));
+	std::auto_ptr<BackupStoreRefCountDatabase> apReferences(
+		BackupStoreRefCountDatabase::Load(
+			apAccounts->GetEntry(0x1234567), true));
+	TEST_EQUAL(subdirid, rRefCount.GetLastObjectIDUsed());
+	TEST_EQUAL(1, rRefCount.GetRefCount(subdirid))
 	
 	// Put more directories in it, if we haven't gone down too far
 	if(depth > 0)
 	{
-		create_test_data_subdirs(protocol, subdirid, "dir_One", depth - 1);
-		create_test_data_subdirs(protocol, subdirid, "dir_Two", depth - 1);
+		create_test_data_subdirs(protocol, subdirid, "dir_One",
+			depth - 1, rRefCount);
+		create_test_data_subdirs(protocol, subdirid, "dir_Two",
+			depth - 1, rRefCount);
 	}
 	
 	// Stick some files in it
-	{
-		BackupStoreFilenameClear name("file_One");
-		std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile("testfiles/file1", subdirid, name));
-		std::auto_ptr<BackupProtocolClientSuccess> stored(protocol.QueryStoreFile(
-			subdirid,
-			0x123456789abcdefLL,		/* modification time */
-			0x7362383249872dfLL,		/* attr hash */
-			0,							/* diff from ID */
-			name,
-			*upload));
-	}
-	{
-		BackupStoreFilenameClear name("file_Two");
-		std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile("testfiles/file1", subdirid, name));
-		std::auto_ptr<BackupProtocolClientSuccess> stored(protocol.QueryStoreFile(
-			subdirid,
-			0x123456789abcdefLL,		/* modification time */
-			0x7362383249872dfLL,		/* attr hash */
-			0,							/* diff from ID */
-			name,
-			*upload));
-	}
-	{
-		BackupStoreFilenameClear name("file_Three");
-		std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile("testfiles/file1", subdirid, name));
-		std::auto_ptr<BackupProtocolClientSuccess> stored(protocol.QueryStoreFile(
-			subdirid,
-			0x123456789abcdefLL,		/* modification time */
-			0x7362383249872dfLL,		/* attr hash */
-			0,							/* diff from ID */
-			name,
-			*upload));
-	}
-
+	create_file_in_dir("file_One", "testfiles/file1", subdirid, protocol,
+		rRefCount);
+	create_file_in_dir("file_Two", "testfiles/file1", subdirid, protocol,
+		rRefCount);
+	create_file_in_dir("file_Three", "testfiles/file1", subdirid, protocol,
+		rRefCount);
 	return subdirid;
 }
 
@@ -885,16 +889,49 @@
 	}
 }
 
-
-int test_server(const char *hostname)
+void init_context(TLSContext& rContext)
 {
-	// Context
-	TLSContext context;
-	context.Initialise(false /* client */,
+	rContext.Initialise(false /* client */,
 			"testfiles/clientCerts.pem",
 			"testfiles/clientPrivKey.pem",
 			"testfiles/clientTrustedCAs.pem");
+}
 
+std::auto_ptr<SocketStreamTLS> open_conn(const char *hostname,
+	TLSContext& rContext)
+{
+	init_context(rContext);
+	std::auto_ptr<SocketStreamTLS> conn(new SocketStreamTLS);
+	conn->Open(rContext, Socket::TypeINET, hostname, BOX_PORT_BBSTORED);
+	return conn;
+}
+
+std::auto_ptr<BackupProtocolClient> test_server_login(SocketStreamTLS& rConn)
+{
+	// Make a protocol
+	std::auto_ptr<BackupProtocolClient> protocol(new
+		BackupProtocolClient(rConn));
+	
+	// Check the version
+	std::auto_ptr<BackupProtocolClientVersion> serverVersion(
+		protocol->QueryVersion(BACKUP_STORE_SERVER_VERSION));
+	TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION);
+
+	// Login
+	std::auto_ptr<BackupProtocolClientLoginConfirmed> loginConf(
+		protocol->QueryLogin(0x01234567, 0));
+
+	return protocol;
+}
+
+int test_server(const char *hostname)
+{
+	TLSContext context;
+	std::auto_ptr<SocketStreamTLS> conn = open_conn(hostname, context);
+	std::auto_ptr<BackupProtocolClient> apProtocol(
+		test_server_login(*conn));
+	BackupProtocolClient& protocol(*apProtocol);
+
 	// Make some test attributes
 	#define ATTR1_SIZE 	245
 	#define ATTR2_SIZE 	23
@@ -911,28 +948,11 @@
 
 	// BLOCK
 	{
-		// Open a connection to the server
-		SocketStreamTLS conn;
-		conn.Open(context, Socket::TypeINET, hostname, BOX_PORT_BBSTORED);
-
-		// Make a protocol
-		BackupProtocolClient protocol(conn);
-		
 		// Get it logging
 		FILE *protocolLog = ::fopen("testfiles/protocol.log", "w");
 		TEST_THAT(protocolLog != 0);
 		protocol.SetLogToFile(protocolLog);
 
-		// Check the version
-		std::auto_ptr<BackupProtocolClientVersion> serverVersion(protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION));
-		TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION);
-
-		// Login
-		std::auto_ptr<BackupProtocolClientLoginConfirmed> loginConf(protocol.QueryLogin(0x01234567, 0));
-		
-		// Check marker is 0
-		TEST_THAT(loginConf->GetClientStoreMarker() == 0);
-
 #ifndef WIN32
 		// Check that we can't open a new connection which requests write permissions
 		{
@@ -1435,10 +1455,18 @@
 		}
 		
 //}	skip:
+
+		std::auto_ptr<BackupStoreAccountDatabase> apAccounts(
+			BackupStoreAccountDatabase::Read(
+				"testfiles/accounts.txt"));
+		std::auto_ptr<BackupStoreRefCountDatabase> apRefCount(
+			BackupStoreRefCountDatabase::Load(
+				apAccounts->GetEntry(0x1234567), true));
 	
 		// Create some nice recursive directories
 		int64_t dirtodelete = create_test_data_subdirs(protocol,
-			BackupProtocolClientListDirectory::RootDirectory, "test_delete", 6 /* depth */);
+			BackupProtocolClientListDirectory::RootDirectory,
+			"test_delete", 6 /* depth */, *apRefCount);
 		
 		// And delete them
 		{
@@ -1760,7 +1788,33 @@
 		TEST_THAT(TestDirExists("testfiles/0_2/backup/01234567"));
 		TEST_THAT(TestGetFileSize("testfiles/accounts.txt") > 8);
 		// make sure something is written to it
+		
+		std::auto_ptr<BackupStoreAccountDatabase> apAccounts(
+			BackupStoreAccountDatabase::Read("testfiles/accounts.txt"));
 
+		std::auto_ptr<BackupStoreRefCountDatabase> apReferences(
+			BackupStoreRefCountDatabase::Load(
+				apAccounts->GetEntry(0x1234567), true));
+		TEST_EQUAL(BACKUPSTORE_ROOT_DIRECTORY_ID,
+			apReferences->GetLastObjectIDUsed());
+		TEST_EQUAL(1, apReferences->GetRefCount(BACKUPSTORE_ROOT_DIRECTORY_ID))
+		apReferences.reset();
+
+		// Delete the refcount database and log in again,
+		// check that it is recreated automatically but with
+		// no objects in it, to ensure seamless upgrade.
+		TEST_EQUAL(0, ::unlink("testfiles/0_0/backup/01234567/refcount.db.rfw"));
+
+		// Context
+		TLSContext context;
+		std::auto_ptr<SocketStreamTLS> conn = open_conn("localhost",
+			context);
+		test_server_login(*conn)->QueryFinished();
+
+		apReferences = BackupStoreRefCountDatabase::Load(
+			apAccounts->GetEntry(0x1234567), true);
+		TEST_EQUAL(0, apReferences->GetLastObjectIDUsed());
+
 		TEST_THAT(ServerIsAlive(pid));
 
 		TEST_THAT(test_server("localhost") == 0);


From boxbackup-dev@boxbackup.org  Sun Jun 28 20:05:08 2009
From: boxbackup-dev@boxbackup.org (boxbackup-dev@boxbackup.org)
Date: Sun, 28 Jun 2009 20:05:08 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2536 - box/trunk/lib/common
Message-ID: <20090628190508.5AD9A325020@www.boxbackup.org>

Author: chris
Date: 2009-06-28 20:05:07 +0100 (Sun, 28 Jun 2009)
New Revision: 2536

Modified:
   box/trunk/lib/common/NamedLock.cpp
   box/trunk/lib/common/NamedLock.h
Log:
Make NamedLock take a std::string instead of a char pointer for C++ 
style.


Modified: box/trunk/lib/common/NamedLock.cpp
===================================================================
--- box/trunk/lib/common/NamedLock.cpp	2009-06-28 12:04:25 UTC (rev 2535)
+++ box/trunk/lib/common/NamedLock.cpp	2009-06-28 19:05:07 UTC (rev 2536)
@@ -65,7 +65,7 @@
 //		Created: 2003/08/28
 //
 // --------------------------------------------------------------------------
-bool NamedLock::TryAndGetLock(const char *Filename, int mode)
+bool NamedLock::TryAndGetLock(const std::string& rFilename, int mode)
 {
 	// Check
 	if(mFileDescriptor != -1)
@@ -75,7 +75,8 @@
 
 	// See if the lock can be got
 #if HAVE_DECL_O_EXLOCK
-	int fd = ::open(Filename, O_WRONLY | O_NONBLOCK | O_CREAT | O_TRUNC | O_EXLOCK, mode);
+	int fd = ::open(rFilename.c_str(),
+		O_WRONLY | O_NONBLOCK | O_CREAT | O_TRUNC | O_EXLOCK, mode);
 	if(fd != -1)
 	{
 		// Got a lock, lovely
@@ -92,10 +93,10 @@
 
 	return false;
 #else
-	int fd = ::open(Filename, O_WRONLY | O_CREAT | O_TRUNC, mode);
+	int fd = ::open(rFilename.c_str(), O_WRONLY | O_CREAT | O_TRUNC, mode);
 	if(fd == -1)
 	{
-		BOX_WARNING("Failed to open lockfile: " << Filename);
+		BOX_WARNING("Failed to open lockfile: " << rFilename);
 		THROW_EXCEPTION(CommonException, OSFileError)
 	}
 

Modified: box/trunk/lib/common/NamedLock.h
===================================================================
--- box/trunk/lib/common/NamedLock.h	2009-06-28 12:04:25 UTC (rev 2535)
+++ box/trunk/lib/common/NamedLock.h	2009-06-28 19:05:07 UTC (rev 2536)
@@ -28,7 +28,7 @@
 	NamedLock(const NamedLock &);
 
 public:
-	bool TryAndGetLock(const char *Filename, int mode = 0755);
+	bool TryAndGetLock(const std::string& rFilename, int mode = 0755);
 	bool GotLock() {return mFileDescriptor != -1;}
 	void ReleaseLock();
 	


From boxbackup-dev@boxbackup.org  Sun Jun 28 20:22:24 2009
From: boxbackup-dev@boxbackup.org (boxbackup-dev@boxbackup.org)
Date: Sun, 28 Jun 2009 20:22:24 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2537 - box/trunk/lib/common
Message-ID: <20090628192224.B47FF325020@www.boxbackup.org>

Author: chris
Date: 2009-06-28 20:22:24 +0100 (Sun, 28 Jun 2009)
New Revision: 2537

Modified:
   box/trunk/lib/common/BoxPortsAndFiles.h.in
Log:
Add a #define for the Box Backup test port, 22011.


Modified: box/trunk/lib/common/BoxPortsAndFiles.h.in
===================================================================
--- box/trunk/lib/common/BoxPortsAndFiles.h.in	2009-06-28 19:05:07 UTC (rev 2536)
+++ box/trunk/lib/common/BoxPortsAndFiles.h.in	2009-06-28 19:22:24 UTC (rev 2537)
@@ -15,6 +15,7 @@
 
 // Backup store daemon
 #define BOX_PORT_BBSTORED			(BOX_PORT_BASE+1)
+#define BOX_PORT_BBSTORED_TEST			22011
 
 // directory within the RAIDFILE root for the backup store daemon
 #define BOX_RAIDFILE_ROOT_BBSTORED		"backup"


From boxbackup-dev@boxbackup.org  Sun Jun 28 20:24:02 2009
From: boxbackup-dev@boxbackup.org (boxbackup-dev@boxbackup.org)
Date: Sun, 28 Jun 2009 20:24:02 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2538 - box/trunk/test/backupstore
Message-ID: <20090628192402.D9D0A325020@www.boxbackup.org>

Author: chris
Date: 2009-06-28 20:24:02 +0100 (Sun, 28 Jun 2009)
New Revision: 2538

Added:
   box/trunk/test/backupstore/Makefile.extra
Log:
Link testbackupstore against HousekeepStoreAccount so that it can call 
housekeeping directly.


Added: box/trunk/test/backupstore/Makefile.extra
===================================================================
--- box/trunk/test/backupstore/Makefile.extra	                        (rev 0)
+++ box/trunk/test/backupstore/Makefile.extra	2009-06-28 19:24:02 UTC (rev 2538)
@@ -0,0 +1 @@
+link-extra: ../../bin/bbstored/HousekeepStoreAccount.o


From boxbackup-dev@boxbackup.org  Sun Jun 28 20:25:25 2009
From: boxbackup-dev@boxbackup.org (boxbackup-dev@boxbackup.org)
Date: Sun, 28 Jun 2009 20:25:25 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2539 - box/trunk/lib/backupstore
Message-ID: <20090628192525.71CE9325020@www.boxbackup.org>

Author: chris
Date: 2009-06-28 20:25:25 +0100 (Sun, 28 Jun 2009)
New Revision: 2539

Modified:
   box/trunk/lib/backupstore/BackupStoreRefCountDatabase.cpp
   box/trunk/lib/backupstore/BackupStoreRefCountDatabase.h
Log:
Make GetRefCount return a refcount_t rather than an int32_t.

Make refcount_t unsigned and make its definition public.


Modified: box/trunk/lib/backupstore/BackupStoreRefCountDatabase.cpp
===================================================================
--- box/trunk/lib/backupstore/BackupStoreRefCountDatabase.cpp	2009-06-28 19:24:02 UTC (rev 2538)
+++ box/trunk/lib/backupstore/BackupStoreRefCountDatabase.cpp	2009-06-28 19:25:25 UTC (rev 2539)
@@ -247,7 +247,8 @@
 //		Created: 2009/06/01
 //
 // --------------------------------------------------------------------------
-int32_t BackupStoreRefCountDatabase::GetRefCount(int64_t ObjectID) const
+BackupStoreRefCountDatabase::refcount_t
+BackupStoreRefCountDatabase::GetRefCount(int64_t ObjectID) const
 {
 	IOStream::pos_type offset = GetOffset(ObjectID);
 

Modified: box/trunk/lib/backupstore/BackupStoreRefCountDatabase.h
===================================================================
--- box/trunk/lib/backupstore/BackupStoreRefCountDatabase.h	2009-06-28 19:24:02 UTC (rev 2538)
+++ box/trunk/lib/backupstore/BackupStoreRefCountDatabase.h	2009-06-28 19:25:25 UTC (rev 2539)
@@ -52,6 +52,8 @@
 {
 	friend class BackupStoreCheck;
 	friend class BackupStoreContext;
+	friend class HousekeepStoreAccount;
+
 public:
 	~BackupStoreRefCountDatabase();
 private:
@@ -73,8 +75,10 @@
 	static std::auto_ptr<BackupStoreRefCountDatabase> Load(const
 		BackupStoreAccountDatabase::Entry& rAccount, bool ReadOnly);
 	
+	typedef uint32_t refcount_t;
+
 	// Data access functions
-	int32_t GetRefCount(int64_t ObjectID) const;
+	refcount_t GetRefCount(int64_t ObjectID) const;
 	int64_t GetLastObjectIDUsed() const;
 	
 	// Data modification functions
@@ -94,8 +98,6 @@
 	static void Create(const BackupStoreAccountDatabase::Entry& rAccount,
 		bool AllowOverwrite);
 
-	typedef int32_t refcount_t;
-
 	static std::string GetFilename(const BackupStoreAccountDatabase::Entry&
 		rAccount);
 	IOStream::pos_type GetSize() const


From boxbackup-dev@boxbackup.org  Sun Jun 28 20:26:19 2009
From: boxbackup-dev@boxbackup.org (boxbackup-dev@boxbackup.org)
Date: Sun, 28 Jun 2009 20:26:19 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2540 - box/trunk/lib/common
Message-ID: <20090628192619.65D6E325020@www.boxbackup.org>

Author: chris
Date: 2009-06-28 20:26:19 +0100 (Sun, 28 Jun 2009)
New Revision: 2540

Modified:
   box/trunk/lib/common/Test.h
Log:
Allow std::ostringstream formatting on the "line" part of 
TEST_EQUAL_LINE, so that it can be used to report object IDs for 
example.


Modified: box/trunk/lib/common/Test.h
===================================================================
--- box/trunk/lib/common/Test.h	2009-06-28 19:25:25 UTC (rev 2539)
+++ box/trunk/lib/common/Test.h	2009-06-28 19:26:19 UTC (rev 2540)
@@ -113,7 +113,9 @@
 	\
 	if(_exp_str != _found_str) \
 	{ \
-		std::string _line_str = _line; \
+		std::ostringstream _ossl; \
+		_ossl << _line; \
+		std::string _line_str = _ossl.str(); \
 		printf("Expected <%s> but found <%s> in <%s>\n", \
 			_exp_str.c_str(), _found_str.c_str(), _line_str.c_str()); \
 		\


From boxbackup-dev@boxbackup.org  Sun Jun 28 20:29:11 2009
From: boxbackup-dev@boxbackup.org (boxbackup-dev@boxbackup.org)
Date: Sun, 28 Jun 2009 20:29:11 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2541 - box/trunk/bin/bbstored
Message-ID: <20090628192911.08D74325020@www.boxbackup.org>

Author: chris
Date: 2009-06-28 20:29:10 +0100 (Sun, 28 Jun 2009)
New Revision: 2541

Modified:
   box/trunk/bin/bbstored/BBStoreDHousekeeping.cpp
   box/trunk/bin/bbstored/BackupStoreDaemon.h
   box/trunk/bin/bbstored/HousekeepStoreAccount.cpp
   box/trunk/bin/bbstored/HousekeepStoreAccount.h
Log:
Make housekeeping check the object reference counts and fix them if 
they're wrong.

Add a callback interface to decouple housekeeping from the 
BackupStoreDaemon, allowing it to be called directly in tests.

Allow housekeeping callers to request it to keep trying forever to get a 
lock on the account if it's busy.


Modified: box/trunk/bin/bbstored/BBStoreDHousekeeping.cpp
===================================================================
--- box/trunk/bin/bbstored/BBStoreDHousekeeping.cpp	2009-06-28 19:26:19 UTC (rev 2540)
+++ box/trunk/bin/bbstored/BBStoreDHousekeeping.cpp	2009-06-28 19:29:10 UTC (rev 2541)
@@ -108,7 +108,8 @@
 				mpAccounts->GetAccountRoot(*i, rootDir, discSet);
 				
 				// Do housekeeping on this account
-				HousekeepStoreAccount housekeeping(*i, rootDir, discSet, *this);
+				HousekeepStoreAccount housekeeping(*i, rootDir,
+					discSet, this);
 				housekeeping.DoHousekeeping();
 			}
 		}

Modified: box/trunk/bin/bbstored/BackupStoreDaemon.h
===================================================================
--- box/trunk/bin/bbstored/BackupStoreDaemon.h	2009-06-28 19:26:19 UTC (rev 2540)
+++ box/trunk/bin/bbstored/BackupStoreDaemon.h	2009-06-28 19:29:10 UTC (rev 2541)
@@ -14,11 +14,11 @@
 #include "BoxPortsAndFiles.h"
 #include "BackupConstants.h"
 #include "BackupStoreContext.h"
+#include "HousekeepStoreAccount.h"
 #include "IOStreamGetLine.h"
 
 class BackupStoreAccounts;
 class BackupStoreAccountDatabase;
-class HousekeepStoreAccount;
 
 // --------------------------------------------------------------------------
 //
@@ -29,10 +29,8 @@
 //
 // --------------------------------------------------------------------------
 class BackupStoreDaemon : public ServerTLS<BOX_PORT_BBSTORED>,
-	HousekeepingInterface
+	HousekeepingInterface, HousekeepingCallback
 {
-	friend class HousekeepStoreAccount;
-
 public:
 	BackupStoreDaemon();
 	~BackupStoreDaemon();
@@ -64,10 +62,13 @@
 	
 	// Housekeeping functions
 	void HousekeepingProcess();
-	bool CheckForInterProcessMsg(int AccountNum = 0, int MaximumWaitTime = 0);
 
 	void LogConnectionStats(const char *commonName, const SocketStreamTLS &s);
 
+public:
+	// HousekeepingInterface implementation
+	virtual bool CheckForInterProcessMsg(int AccountNum = 0, int MaximumWaitTime = 0);
+
 private:
 	BackupStoreAccountDatabase *mpAccountDatabase;
 	BackupStoreAccounts *mpAccounts;

Modified: box/trunk/bin/bbstored/HousekeepStoreAccount.cpp
===================================================================
--- box/trunk/bin/bbstored/HousekeepStoreAccount.cpp	2009-06-28 19:26:19 UTC (rev 2540)
+++ box/trunk/bin/bbstored/HousekeepStoreAccount.cpp	2009-06-28 19:29:10 UTC (rev 2541)
@@ -39,11 +39,13 @@
 //		Created: 11/12/03
 //
 // --------------------------------------------------------------------------
-HousekeepStoreAccount::HousekeepStoreAccount(int AccountID, const std::string &rStoreRoot, int StoreDiscSet, BackupStoreDaemon &rDaemon)
+HousekeepStoreAccount::HousekeepStoreAccount(int AccountID,
+	const std::string &rStoreRoot, int StoreDiscSet,
+	HousekeepingCallback* pHousekeepingCallback)
 	: mAccountID(AccountID),
 	  mStoreRoot(rStoreRoot),
 	  mStoreDiscSet(StoreDiscSet),
-	  mrDaemon(rDaemon),
+	  mpHousekeepingCallback(pHousekeepingCallback),
 	  mDeletionSizeTarget(0),
   	  mPotentialDeletionsTotalSize(0),
 	  mMaxSizeInPotentialDeletions(0),
@@ -57,6 +59,7 @@
 	  mBlocksInDirectoriesDelta(0),
 	  mFilesDeleted(0),
 	  mEmptyDirectoriesDeleted(0),
+	  mSuppressRefCountChangeWarnings(false),
 	  mCountUntilNextInterprocessMsgCheck(POLL_INTERPROCESS_MSG_CHECK_FREQUENCY)
 {
 }
@@ -81,7 +84,7 @@
 //		Created: 11/12/03
 //
 // --------------------------------------------------------------------------
-void HousekeepStoreAccount::DoHousekeeping()
+void HousekeepStoreAccount::DoHousekeeping(bool KeepTryingForever)
 {
 	// Attempt to lock the account
 	std::string writeLockFilename;
@@ -91,8 +94,21 @@
 	if(!writeLock.TryAndGetLock(writeLockFilename.c_str(),
 		0600 /* restrictive file permissions */))
 	{
-		// Couldn't lock the account -- just stop now
-		return;
+		if(KeepTryingForever)
+		{
+			BOX_WARNING("Failed to lock account for housekeeping, "
+				"still trying...");
+			while(!writeLock.TryAndGetLock(writeLockFilename,
+				0600 /* restrictive file permissions */))
+			{
+				sleep(1);
+			}
+		}
+		else
+		{
+			// Couldn't lock the account -- just stop now
+			return;
+		}
 	}
 
 	// Load the store info to find necessary info for the housekeeping
@@ -106,6 +122,14 @@
 		mDeletionSizeTarget = 0;
 	}
 
+	// initialise the refcount database
+	mNewRefCounts.clear();
+	// try to pre-allocate as much memory as we need
+	mNewRefCounts.reserve(info->GetLastObjectIDUsed());
+	// initialise the refcount of the root entry
+	mNewRefCounts.resize(BACKUPSTORE_ROOT_DIRECTORY_ID + 1, 0);
+	mNewRefCounts[BACKUPSTORE_ROOT_DIRECTORY_ID] = 1;
+
 	// Scan the directory for potential things to delete
 	// This will also remove eligible items marked with RemoveASAP
 	bool continueHousekeeping = ScanDirectory(BACKUPSTORE_ROOT_DIRECTORY_ID);
@@ -207,6 +231,86 @@
 			mEmptyDirectoriesDeleted << " dirs)" <<
 			(deleteInterrupted?" and was interrupted":""));
 	}
+
+	// We can only update the refcount database if we successfully
+	// finished our scan of all directories, otherwise we don't actually
+	// know which of the new counts are valid and which aren't
+	// (we might not have seen second references to some objects, etc.)
+
+	BackupStoreAccountDatabase::Entry account(mAccountID, mStoreDiscSet);
+	std::auto_ptr<BackupStoreRefCountDatabase> apReferences;
+
+	// try to load the reference count database
+	try
+	{
+		apReferences = BackupStoreRefCountDatabase::Load(account,
+			false);
+	}
+	catch(BoxException &e)
+	{
+		BOX_WARNING("Reference count database is missing or corrupted "
+			"during housekeeping, creating a new one.");
+		mSuppressRefCountChangeWarnings = true;
+		BackupStoreRefCountDatabase::CreateForRegeneration(account);
+		apReferences = BackupStoreRefCountDatabase::Load(account,
+			false);
+	}
+
+	int64_t LastUsedObjectIdOnDisk = apReferences->GetLastObjectIDUsed();
+
+	for (int64_t ObjectID = BACKUPSTORE_ROOT_DIRECTORY_ID;
+		ObjectID < mNewRefCounts.size(); ObjectID++)
+	{
+		if (ObjectID > LastUsedObjectIdOnDisk)
+		{
+			if (!mSuppressRefCountChangeWarnings)
+			{
+				BOX_WARNING("Reference count of object " <<
+					BOX_FORMAT_OBJECTID(ObjectID) <<
+					" not found in database, added"
+					" with " << mNewRefCounts[ObjectID] <<
+					" references");
+			}
+			apReferences->SetRefCount(ObjectID,
+				mNewRefCounts[ObjectID]);
+			LastUsedObjectIdOnDisk = ObjectID;
+			continue;
+		}
+
+		BackupStoreRefCountDatabase::refcount_t OldRefCount =
+			apReferences->GetRefCount(ObjectID);
+
+		if (OldRefCount != mNewRefCounts[ObjectID])
+		{
+			BOX_WARNING("Reference count of object " <<
+				BOX_FORMAT_OBJECTID(ObjectID) <<
+				" changed from " << OldRefCount <<
+				" to " << mNewRefCounts[ObjectID]);
+			apReferences->SetRefCount(ObjectID,
+				mNewRefCounts[ObjectID]);
+		}
+	}
+
+	// zero excess references in the database
+	for (int64_t ObjectID = mNewRefCounts.size(); 
+		ObjectID <= LastUsedObjectIdOnDisk; ObjectID++)
+	{
+		BackupStoreRefCountDatabase::refcount_t OldRefCount =
+			apReferences->GetRefCount(ObjectID);
+		BackupStoreRefCountDatabase::refcount_t NewRefCount = 0;
+
+		if (OldRefCount != NewRefCount)
+		{
+			BOX_WARNING("Reference count of object " <<
+				BOX_FORMAT_OBJECTID(ObjectID) <<
+				" changed from " << OldRefCount <<
+				" to " << NewRefCount << " (not found)");
+			apReferences->SetRefCount(ObjectID, NewRefCount);
+		}
+	}
+
+	// force file to be saved and closed before releasing the lock below
+	apReferences.reset();
 	
 	// Make sure the delta's won't cause problems if the counts are 
 	// really wrong, and it wasn't fixed because the store was 
@@ -279,7 +383,7 @@
 
 		// Check for having to stop
 		// Include account ID here as the specified account is locked
-		if(mrDaemon.CheckForInterProcessMsg(mAccountID))
+		if(mpHousekeepingCallback && mpHousekeepingCallback->CheckForInterProcessMsg(mAccountID))
 		{
 			// Need to abort now
 			return false;
@@ -359,6 +463,13 @@
 
 		while((en = i.Next(BackupStoreDirectory::Entry::Flags_File)) != 0)
 		{
+			// This directory references this object
+			if (mNewRefCounts.size() <= en->GetObjectID())
+			{
+				mNewRefCounts.resize(en->GetObjectID() + 1, 0);
+			}
+			mNewRefCounts[en->GetObjectID()]++;
+
 			// Update recalculated usage sizes
 			int16_t enFlags = en->GetFlags();
 			int64_t enSizeInBlocks = en->GetSizeInBlocks();
@@ -467,6 +578,13 @@
 		BackupStoreDirectory::Entry *en = 0;
 		while((en = i.Next(BackupStoreDirectory::Entry::Flags_Dir)) != 0)
 		{
+			// This parent directory references this child
+			if (mNewRefCounts.size() <= en->GetObjectID())
+			{
+				mNewRefCounts.resize(en->GetObjectID() + 1, 0);
+			}
+			mNewRefCounts[en->GetObjectID()]++;
+
 			// Next level
 			ASSERT((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) == BackupStoreDirectory::Entry::Flags_Dir);
 			
@@ -551,7 +669,7 @@
 		{
 			mCountUntilNextInterprocessMsgCheck = POLL_INTERPROCESS_MSG_CHECK_FREQUENCY;
 			// Check for having to stop
-			if(mrDaemon.CheckForInterProcessMsg(mAccountID))	// include account ID here as the specified account is now locked
+			if(mpHousekeepingCallback && mpHousekeepingCallback->CheckForInterProcessMsg(mAccountID))	// include account ID here as the specified account is now locked
 			{
 				// Need to abort now
 				return true;
@@ -808,7 +926,7 @@
 			{
 				mCountUntilNextInterprocessMsgCheck = POLL_INTERPROCESS_MSG_CHECK_FREQUENCY;
 				// Check for having to stop
-				if(mrDaemon.CheckForInterProcessMsg(mAccountID))	// include account ID here as the specified account is now locked
+				if(mpHousekeepingCallback && mpHousekeepingCallback->CheckForInterProcessMsg(mAccountID))	// include account ID here as the specified account is now locked
 				{
 					// Need to abort now
 					return true;

Modified: box/trunk/bin/bbstored/HousekeepStoreAccount.h
===================================================================
--- box/trunk/bin/bbstored/HousekeepStoreAccount.h	2009-06-28 19:26:19 UTC (rev 2540)
+++ box/trunk/bin/bbstored/HousekeepStoreAccount.h	2009-06-28 19:29:10 UTC (rev 2541)
@@ -17,6 +17,12 @@
 class BackupStoreDaemon;
 class BackupStoreDirectory;
 
+class HousekeepingCallback
+{
+	public:
+	virtual ~HousekeepingCallback() {}
+	virtual bool CheckForInterProcessMsg(int AccountNum = 0, int MaximumWaitTime = 0) = 0;
+};
 
 // --------------------------------------------------------------------------
 //
@@ -29,10 +35,11 @@
 class HousekeepStoreAccount
 {
 public:
-	HousekeepStoreAccount(int AccountID, const std::string &rStoreRoot, int StoreDiscSet, BackupStoreDaemon &rDaemon);
+	HousekeepStoreAccount(int AccountID, const std::string &rStoreRoot,
+		int StoreDiscSet, HousekeepingCallback* pHousekeepingCallback);
 	~HousekeepStoreAccount();
 	
-	void DoHousekeeping();
+	void DoHousekeeping(bool KeepTryingForever = false);
 	
 	
 private:
@@ -65,7 +72,7 @@
 	int mAccountID;
 	std::string mStoreRoot;
 	int mStoreDiscSet;
-	BackupStoreDaemon &mrDaemon;
+	HousekeepingCallback* mpHousekeepingCallback;
 	
 	int64_t mDeletionSizeTarget;
 	
@@ -91,6 +98,10 @@
 	// Deletion count
 	int64_t mFilesDeleted;
 	int64_t mEmptyDirectoriesDeleted;
+
+	// New reference count list
+	std::vector<uint32_t> mNewRefCounts;
+	bool mSuppressRefCountChangeWarnings;
 	
 	// Poll frequency
 	int mCountUntilNextInterprocessMsgCheck;


From boxbackup-dev@boxbackup.org  Sun Jun 28 20:33:35 2009
From: boxbackup-dev@boxbackup.org (boxbackup-dev@boxbackup.org)
Date: Sun, 28 Jun 2009 20:33:35 +0100 (BST)
Subject: [Box Backup-commit] COMMIT r2542 - in box/trunk/test/backupstore: . testfiles
Message-ID: <20090628193335.0AB78325020@www.boxbackup.org>

Author: chris
Date: 2009-06-28 20:33:34 +0100 (Sun, 28 Jun 2009)
New Revision: 2542

Modified:
   box/trunk/test/backupstore/testbackupstore.cpp
   box/trunk/test/backupstore/testfiles/bbstored.conf
Log:
Test that housekeeping can recreate the refcount database if it's 
missing.


Modified: box/trunk/test/backupstore/testbackupstore.cpp
===================================================================
--- box/trunk/test/backupstore/testbackupstore.cpp	2009-06-28 19:29:10 UTC (rev 2541)
+++ box/trunk/test/backupstore/testbackupstore.cpp	2009-06-28 19:33:34 UTC (rev 2542)
@@ -36,6 +36,8 @@
 #include "ServerControl.h"
 #include "BackupStoreAccountDatabase.h"
 #include "BackupStoreRefCountDatabase.h"
+#include "BackupStoreAccounts.h"
+#include "HousekeepStoreAccount.h"
 
 #include "MemLeakFindOn.h"
 
@@ -472,6 +474,17 @@
 	TEST_THAT(dirs == 0 || dirs == 2);
 }
 
+std::vector<uint32_t> ExpectedRefCounts;
+
+void set_refcount(int64_t ObjectID, uint32_t RefCount = 1)
+{
+	if (ExpectedRefCounts.size() <= ObjectID);
+	{
+		ExpectedRefCounts.resize(ObjectID + 1, 0);
+	}
+	ExpectedRefCounts[ObjectID] = RefCount;
+}
+
 void create_file_in_dir(std::string name, std::string source, int64_t parentId,
 	BackupProtocolClient &protocol, BackupStoreRefCountDatabase& rRefCount)
 {
@@ -489,6 +502,7 @@
 	int64_t objectId = stored->GetObjectID();
 	TEST_EQUAL(objectId, rRefCount.GetLastObjectIDUsed());
 	TEST_EQUAL(1, rRefCount.GetRefCount(objectId))
+	set_refcount(objectId, 1);
 }
 
 int64_t create_test_data_subdirs(BackupProtocolClient &protocol, int64_t indir,
@@ -509,13 +523,9 @@
 	
 	printf("Create subdirs, depth = %d, dirid = %llx\n", depth, subdirid);
 
-	std::auto_ptr<BackupStoreAccountDatabase> apAccounts(
-		BackupStoreAccountDatabase::Read("testfiles/accounts.txt"));
-	std::auto_ptr<BackupStoreRefCountDatabase> apReferences(
-		BackupStoreRefCountDatabase::Load(
-			apAccounts->GetEntry(0x1234567), true));
 	TEST_EQUAL(subdirid, rRefCount.GetLastObjectIDUsed());
 	TEST_EQUAL(1, rRefCount.GetRefCount(subdirid))
+	set_refcount(subdirid, 1);
 	
 	// Put more directories in it, if we haven't gone down too far
 	if(depth > 0)
@@ -634,7 +644,8 @@
 
 	// Get a connection
 	SocketStreamTLS connReadOnly;
-	connReadOnly.Open(context, Socket::TypeINET, hostname, BOX_PORT_BBSTORED);
+	connReadOnly.Open(context, Socket::TypeINET, hostname,
+		BOX_PORT_BBSTORED_TEST);
 	BackupProtocolClient protocolReadOnly(connReadOnly);
 
 	{
@@ -796,6 +807,7 @@
 		store1objid = stored->GetObjectID();
 		TEST_THAT(store1objid == 2);
 	}
+	set_refcount(store1objid, 1);
 	// And retrieve it
 	{
 		// Retrieve as object
@@ -902,7 +914,8 @@
 {
 	init_context(rContext);
 	std::auto_ptr<SocketStreamTLS> conn(new SocketStreamTLS);
-	conn->Open(rContext, Socket::TypeINET, hostname, BOX_PORT_BBSTORED);
+	conn->Open(rContext, Socket::TypeINET, hostname,
+		BOX_PORT_BBSTORED_TEST);
 	return conn;
 }
 
@@ -924,6 +937,17 @@
 	return protocol;
 }
 
+void run_housekeeping(BackupStoreAccountDatabase::Entry& rAccount)
+{
+	std::string rootDir = BackupStoreAccounts::GetAccountRoot(rAccount);
+	int discSet = rAccount.GetDiscSet();
+
+	// Do housekeeping on this account
+	HousekeepStoreAccount housekeeping(rAccount.GetID(), rootDir,
+		discSet, NULL);
+	housekeeping.DoHousekeeping(true /* keep trying forever */);
+}
+
 int test_server(const char *hostname)
 {
 	TLSContext context;
@@ -957,7 +981,8 @@
 		// Check that we can't open a new connection which requests write permissions
 		{
 			SocketStreamTLS conn;
-			conn.Open(context, Socket::TypeINET, hostname, BOX_PORT_BBSTORED);
+			conn.Open(context, Socket::TypeINET, hostname,
+				BOX_PORT_BBSTORED_TEST);
 			BackupProtocolClient protocol(conn);
 			std::auto_ptr<BackupProtocolClientVersion> serverVersion(protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION));
 			TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION);
@@ -973,7 +998,8 @@
 #ifndef WIN32
 		// Open a new connection which is read only
 		SocketStreamTLS connReadOnly;
-		connReadOnly.Open(context, Socket::TypeINET, hostname, BOX_PORT_BBSTORED);
+		connReadOnly.Open(context, Socket::TypeINET, hostname,
+			BOX_PORT_BBSTORED_TEST);
 		BackupProtocolClient protocolReadOnly(connReadOnly);
 
 		// Get it logging
@@ -1021,6 +1047,10 @@
 			uploads[t].allocated_objid = stored->GetObjectID();
 			uploads[t].mod_time = modtime;
 			if(maxID < stored->GetObjectID()) maxID = stored->GetObjectID();
+			set_refcount(stored->GetObjectID(), 1);
+			BOX_TRACE("wrote file " << filename << " to server "
+				"as object " <<
+				BOX_FORMAT_OBJECTID(stored->GetObjectID()));
 		}
 
 		// Add some attributes onto one of them
@@ -1141,6 +1171,9 @@
 				if(maxID < stored->GetObjectID()) maxID = stored->GetObjectID();
 				patchedID = stored->GetObjectID();
 			}
+
+			set_refcount(patchedID, 1);
+
 			// Then download it to check it's OK
 			std::auto_ptr<BackupProtocolClientSuccess> getFile(protocol.QueryGetFile(BackupProtocolClientListDirectory::RootDirectory, patchedID));
 			TEST_THAT(getFile->GetObjectID() == patchedID);
@@ -1162,6 +1195,9 @@
 			subdirid = dirCreate->GetObjectID(); 
 			TEST_THAT(subdirid == maxID + 1);
 		}
+
+		set_refcount(subdirid, 1);
+
 		// Stick a file in it
 		int64_t subdirfileid = 0;
 		{
@@ -1179,6 +1215,8 @@
 			subdirfileid = stored->GetObjectID();
 		}
 
+		set_refcount(subdirfileid, 1);
+
 		printf("\n==== Checking upload using read-only connection\n");
 		// Check the directories on the read only connection
 		{
@@ -1402,6 +1440,9 @@
 			subsubfileid = stored->GetObjectID();
 		}
 
+		set_refcount(subsubdirid, 1);
+		set_refcount(subsubfileid, 1);
+
 		// Query names -- test that invalid stuff returns not found OK
 		{
 			std::auto_ptr<BackupProtocolClientObjectName> nameRep(protocol.QueryGetObjectName(3248972347823478927LL, subsubdirid));
@@ -1759,7 +1800,8 @@
 		{
 			// Open a connection to the server
 			SocketStreamTLS conn;
-			conn.Open(context, Socket::TypeINET, "localhost", BOX_PORT_BBSTORED);
+			conn.Open(context, Socket::TypeINET, "localhost",
+				BOX_PORT_BBSTORED_TEST);
 
 			// Make a protocol
 			BackupProtocolClient protocol(conn);
@@ -1805,19 +1847,61 @@
 		// no objects in it, to ensure seamless upgrade.
 		TEST_EQUAL(0, ::unlink("testfiles/0_0/backup/01234567/refcount.db.rfw"));
 
-		// Context
 		TLSContext context;
 		std::auto_ptr<SocketStreamTLS> conn = open_conn("localhost",
 			context);
 		test_server_login(*conn)->QueryFinished();
 
-		apReferences = BackupStoreRefCountDatabase::Load(
-			apAccounts->GetEntry(0x1234567), true);
+		BackupStoreAccountDatabase::Entry account =
+			apAccounts->GetEntry(0x1234567);
+		apReferences = BackupStoreRefCountDatabase::Load(account, true);
 		TEST_EQUAL(0, apReferences->GetLastObjectIDUsed());
 
 		TEST_THAT(ServerIsAlive(pid));
 
+		run_housekeeping(account);
+
+		// Check that housekeeping fixed the ref counts
+		TEST_EQUAL(BACKUPSTORE_ROOT_DIRECTORY_ID,
+			apReferences->GetLastObjectIDUsed());
+		TEST_EQUAL(1, apReferences->GetRefCount(BACKUPSTORE_ROOT_DIRECTORY_ID))
+
+		TEST_THAT(ServerIsAlive(pid));
+
+		set_refcount(BACKUPSTORE_ROOT_DIRECTORY_ID, 1);
+
 		TEST_THAT(test_server("localhost") == 0);
+
+		// test that all object reference counts have the
+		// expected values
+		TEST_EQUAL(ExpectedRefCounts.size() - 1,
+			apReferences->GetLastObjectIDUsed());
+		for (unsigned int i = BACKUPSTORE_ROOT_DIRECTORY_ID;
+			i < ExpectedRefCounts.size(); i++)
+		{
+			TEST_EQUAL_LINE(ExpectedRefCounts[i],
+				apReferences->GetRefCount(i),
+				"object " << BOX_FORMAT_OBJECTID(i));
+		}
+
+		// Delete the refcount database again, and let
+		// housekeeping recreate it and fix the ref counts.
+		// This could also happen after upgrade, if a housekeeping
+		// runs before the user logs in.
+		apReferences.reset();
+		TEST_EQUAL(0, ::unlink("testfiles/0_0/backup/01234567/refcount.db.rfw"));
+		run_housekeeping(account);
+		apReferences = BackupStoreRefCountDatabase::Load(account, true);
+
+		TEST_EQUAL(ExpectedRefCounts.size() - 1,
+			apReferences->GetLastObjectIDUsed());
+		for (unsigned int i = BACKUPSTORE_ROOT_DIRECTORY_ID;
+			i < ExpectedRefCounts.size(); i++)
+		{
+			TEST_EQUAL_LINE(ExpectedRefCounts[i],
+				apReferences->GetRefCount(i),
+				"object " << BOX_FORMAT_OBJECTID(i));
+		}
 		
 		// Test the deletion of objects by the housekeeping system
 		// First, things as they are now.
@@ -1886,7 +1970,8 @@
 		{
 			// Open a connection to the server
 			SocketStreamTLS conn;
-			conn.Open(context, Socket::TypeINET, "localhost", BOX_PORT_BBSTORED);
+			conn.Open(context, Socket::TypeINET, "localhost",
+				BOX_PORT_BBSTORED_TEST);
 
 			// Make a protocol
 			BackupProtocolClient protocol(conn);

Modified: box/trunk/test/backupstore/testfiles/bbstored.conf
===================================================================
--- box/trunk/test/backupstore/testfiles/bbstored.conf	2009-06-28 19:29:10 UTC (rev 2541)
+++ box/trunk/test/backupstore/testfiles/bbstored.conf	2009-06-28 19:33:34 UTC (rev 2542)
@@ -9,7 +9,7 @@
 Server
 {
 	PidFile = testfiles/bbstored.pid
-	ListenAddresses = inet:localhost
+	ListenAddresses = inet:localhost:22011
 	CertificateFile = testfiles/serverCerts.pem
 	PrivateKeyFile = testfiles/serverPrivKey.pem
 	TrustedCAsFile = testfiles/serverTrustedCAs.pem


