[Box Backup-commit] COMMIT r2535 - in box/trunk: bin/bbstored test/backupstore
boxbackup-dev@boxbackup.org
boxbackup-dev@boxbackup.org
Sun, 28 Jun 2009 13:04:26 +0100 (BST)
Author: chris
Date: 2009-06-28 13:04:25 +0100 (Sun, 28 Jun 2009)
New Revision: 2535
Modified:
box/trunk/bin/bbstored/BackupStoreContext.cpp
box/trunk/bin/bbstored/BackupStoreContext.h
box/trunk/test/backupstore/testbackupstore.cpp
Log:
Update reference counts when files and directories are uploaded.
Modified: box/trunk/bin/bbstored/BackupStoreContext.cpp
===================================================================
--- box/trunk/bin/bbstored/BackupStoreContext.cpp 2009-06-27 11:38:52 UTC (rev 2534)
+++ box/trunk/bin/bbstored/BackupStoreContext.cpp 2009-06-28 12:04:25 UTC (rev 2535)
@@ -191,6 +191,23 @@
// Keep the pointer to it
mpStoreInfo = i;
+
+ BackupStoreAccountDatabase::Entry account(mClientID, mStoreDiscSet);
+
+ // try to load the reference count database
+ try
+ {
+ mapRefCount = BackupStoreRefCountDatabase::Load(account, false);
+ }
+ catch(BoxException &e)
+ {
+ BOX_WARNING("Reference count database is missing or corrupted, "
+ "creating a new one, expect housekeeping to find and "
+ "fix problems with reference counts later.");
+
+ BackupStoreRefCountDatabase::CreateForRegeneration(account);
+ mapRefCount = BackupStoreRefCountDatabase::Load(account, false);
+ }
}
@@ -395,15 +412,18 @@
// --------------------------------------------------------------------------
//
// Function
-// Name: BackupStoreContext::AddFile(IOStream &, int64_t, int64_t, int64_t, const BackupStoreFilename &, bool)
-// Purpose: Add a file to the store, from a given stream, into a specified directory.
-// Returns object ID of the new file.
+// Name: BackupStoreContext::AddFile(IOStream &, int64_t,
+// int64_t, int64_t, const BackupStoreFilename &, bool)
+// Purpose: Add a file to the store, from a given stream, into
+// a specified directory. Returns object ID of the new
+// file.
// Created: 2003/09/03
//
// --------------------------------------------------------------------------
-int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, int64_t ModificationTime,
- int64_t AttributesHash, int64_t DiffFromFileID, const BackupStoreFilename &rFilename,
- bool MarkFileWithSameNameAsOldVersions)
+int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory,
+ int64_t ModificationTime, int64_t AttributesHash,
+ int64_t DiffFromFileID, const BackupStoreFilename &rFilename,
+ bool MarkFileWithSameNameAsOldVersions)
{
if(mpStoreInfo.get() == 0)
{
@@ -670,6 +690,9 @@
mpStoreInfo->ChangeBlocksUsed(blocksUsed);
mpStoreInfo->ChangeBlocksInOldFiles(blocksInOldFiles);
+ // Increment reference count on the new directory to one
+ mapRefCount->AddReference(id);
+
// Save the store info -- can cope if this exceptions because infomation
// will be rebuilt by housekeeping, and ID allocation can recover.
SaveStoreInfo();
@@ -768,8 +791,9 @@
// --------------------------------------------------------------------------
//
// Function
-// Name: BackupStoreContext::DeleteFile(const BackupStoreFilename &, int64_t, int64_t &)
-// Purpose: Deletes a file, returning true if the file existed. Object ID returned too, set to zero if not found.
+// Name: BackupStoreContext::UndeleteFile(int64_t, int64_t)
+// Purpose: Undeletes a file, if it exists, returning true if
+// the file existed.
// Created: 2003/10/21
//
// --------------------------------------------------------------------------
@@ -933,8 +957,10 @@
// --------------------------------------------------------------------------
//
// Function
-// Name: BackupStoreContext::AddDirectory(int64_t, const BackupStoreFilename &, bool &)
-// Purpose: Creates a directory (or just returns the ID of an existing one). rAlreadyExists set appropraitely.
+// Name: BackupStoreContext::AddDirectory(int64_t,
+// const BackupStoreFilename &, bool &)
+// Purpose: Creates a directory (or just returns the ID of an
+// existing one). rAlreadyExists set appropraitely.
// Created: 2003/09/04
//
// --------------------------------------------------------------------------
@@ -974,7 +1000,7 @@
// Allocate the next ID
int64_t id = AllocateObjectID();
- // Create a blank directory with the given attributes on disc
+ // Create an empty directory with the given attributes on disc
std::string fn;
MakeObjectFilename(id, fn, true /* make sure the directory it's in exists */);
{
@@ -998,11 +1024,14 @@
// Not added to cache, so don't set the size in the directory
}
- // Then add it into the directory
+ // Then add it into the parent directory
try
{
dir.AddEntry(rFilename, 0 /* modification time */, id, 0 /* blocks used */, BackupStoreDirectory::Entry::Flags_Dir, 0 /* attributes mod time */);
SaveDirectory(dir, InDirectory);
+
+ // Increment reference count on the new directory to one
+ mapRefCount->AddReference(id);
}
catch(...)
{
@@ -1016,7 +1045,7 @@
// Don't worry about the incremented number in the store info
throw;
}
-
+
// Save the store info (may be postponed)
SaveStoreInfo();
Modified: box/trunk/bin/bbstored/BackupStoreContext.h
===================================================================
--- box/trunk/bin/bbstored/BackupStoreContext.h 2009-06-27 11:38:52 UTC (rev 2534)
+++ box/trunk/bin/bbstored/BackupStoreContext.h 2009-06-28 12:04:25 UTC (rev 2535)
@@ -14,6 +14,7 @@
#include <map>
#include <memory>
+#include "BackupStoreRefCountDatabase.h"
#include "NamedLock.h"
#include "ProtocolObject.h"
#include "Utils.h"
@@ -137,7 +138,6 @@
void DeleteDirectoryRecurse(int64_t ObjectID, int64_t &rBlocksDeletedOut, bool Undelete);
int64_t AllocateObjectID();
-private:
int32_t mClientID;
HousekeepingInterface &mrDaemon;
int mProtocolPhase;
@@ -150,6 +150,9 @@
// Store info
std::auto_ptr<BackupStoreInfo> mpStoreInfo;
+
+ // Refcount database
+ std::auto_ptr<BackupStoreRefCountDatabase> mapRefCount;
// Directory cache
std::map<int64_t, BackupStoreDirectory*> mDirectoryCache;
Modified: box/trunk/test/backupstore/testbackupstore.cpp
===================================================================
--- box/trunk/test/backupstore/testbackupstore.cpp 2009-06-27 11:38:52 UTC (rev 2534)
+++ box/trunk/test/backupstore/testbackupstore.cpp 2009-06-28 12:04:25 UTC (rev 2535)
@@ -34,6 +34,8 @@
#include "BackupClientFileAttributes.h"
#include "BackupClientCryptoKeys.h"
#include "ServerControl.h"
+#include "BackupStoreAccountDatabase.h"
+#include "BackupStoreRefCountDatabase.h"
#include "MemLeakFindOn.h"
@@ -470,8 +472,28 @@
TEST_THAT(dirs == 0 || dirs == 2);
}
-int64_t create_test_data_subdirs(BackupProtocolClient &protocol, int64_t indir, const char *name, int depth)
+void create_file_in_dir(std::string name, std::string source, int64_t parentId,
+ BackupProtocolClient &protocol, BackupStoreRefCountDatabase& rRefCount)
{
+ BackupStoreFilenameClear name_encoded("file_One");
+ std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile(
+ source.c_str(), parentId, name_encoded));
+ std::auto_ptr<BackupProtocolClientSuccess> stored(
+ protocol.QueryStoreFile(
+ parentId,
+ 0x123456789abcdefLL, /* modification time */
+ 0x7362383249872dfLL, /* attr hash */
+ 0, /* diff from ID */
+ name_encoded,
+ *upload));
+ int64_t objectId = stored->GetObjectID();
+ TEST_EQUAL(objectId, rRefCount.GetLastObjectIDUsed());
+ TEST_EQUAL(1, rRefCount.GetRefCount(objectId))
+}
+
+int64_t create_test_data_subdirs(BackupProtocolClient &protocol, int64_t indir,
+ const char *name, int depth, BackupStoreRefCountDatabase& rRefCount)
+{
// Create a directory
int64_t subdirid = 0;
BackupStoreFilenameClear dirname(name);
@@ -486,49 +508,31 @@
}
printf("Create subdirs, depth = %d, dirid = %llx\n", depth, subdirid);
+
+ std::auto_ptr<BackupStoreAccountDatabase> apAccounts(
+ BackupStoreAccountDatabase::Read("testfiles/accounts.txt"));
+ std::auto_ptr<BackupStoreRefCountDatabase> apReferences(
+ BackupStoreRefCountDatabase::Load(
+ apAccounts->GetEntry(0x1234567), true));
+ TEST_EQUAL(subdirid, rRefCount.GetLastObjectIDUsed());
+ TEST_EQUAL(1, rRefCount.GetRefCount(subdirid))
// Put more directories in it, if we haven't gone down too far
if(depth > 0)
{
- create_test_data_subdirs(protocol, subdirid, "dir_One", depth - 1);
- create_test_data_subdirs(protocol, subdirid, "dir_Two", depth - 1);
+ create_test_data_subdirs(protocol, subdirid, "dir_One",
+ depth - 1, rRefCount);
+ create_test_data_subdirs(protocol, subdirid, "dir_Two",
+ depth - 1, rRefCount);
}
// Stick some files in it
- {
- BackupStoreFilenameClear name("file_One");
- std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile("testfiles/file1", subdirid, name));
- std::auto_ptr<BackupProtocolClientSuccess> stored(protocol.QueryStoreFile(
- subdirid,
- 0x123456789abcdefLL, /* modification time */
- 0x7362383249872dfLL, /* attr hash */
- 0, /* diff from ID */
- name,
- *upload));
- }
- {
- BackupStoreFilenameClear name("file_Two");
- std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile("testfiles/file1", subdirid, name));
- std::auto_ptr<BackupProtocolClientSuccess> stored(protocol.QueryStoreFile(
- subdirid,
- 0x123456789abcdefLL, /* modification time */
- 0x7362383249872dfLL, /* attr hash */
- 0, /* diff from ID */
- name,
- *upload));
- }
- {
- BackupStoreFilenameClear name("file_Three");
- std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile("testfiles/file1", subdirid, name));
- std::auto_ptr<BackupProtocolClientSuccess> stored(protocol.QueryStoreFile(
- subdirid,
- 0x123456789abcdefLL, /* modification time */
- 0x7362383249872dfLL, /* attr hash */
- 0, /* diff from ID */
- name,
- *upload));
- }
-
+ create_file_in_dir("file_One", "testfiles/file1", subdirid, protocol,
+ rRefCount);
+ create_file_in_dir("file_Two", "testfiles/file1", subdirid, protocol,
+ rRefCount);
+ create_file_in_dir("file_Three", "testfiles/file1", subdirid, protocol,
+ rRefCount);
return subdirid;
}
@@ -885,16 +889,49 @@
}
}
-
-int test_server(const char *hostname)
+void init_context(TLSContext& rContext)
{
- // Context
- TLSContext context;
- context.Initialise(false /* client */,
+ rContext.Initialise(false /* client */,
"testfiles/clientCerts.pem",
"testfiles/clientPrivKey.pem",
"testfiles/clientTrustedCAs.pem");
+}
+std::auto_ptr<SocketStreamTLS> open_conn(const char *hostname,
+ TLSContext& rContext)
+{
+ init_context(rContext);
+ std::auto_ptr<SocketStreamTLS> conn(new SocketStreamTLS);
+ conn->Open(rContext, Socket::TypeINET, hostname, BOX_PORT_BBSTORED);
+ return conn;
+}
+
+std::auto_ptr<BackupProtocolClient> test_server_login(SocketStreamTLS& rConn)
+{
+ // Make a protocol
+ std::auto_ptr<BackupProtocolClient> protocol(new
+ BackupProtocolClient(rConn));
+
+ // Check the version
+ std::auto_ptr<BackupProtocolClientVersion> serverVersion(
+ protocol->QueryVersion(BACKUP_STORE_SERVER_VERSION));
+ TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION);
+
+ // Login
+ std::auto_ptr<BackupProtocolClientLoginConfirmed> loginConf(
+ protocol->QueryLogin(0x01234567, 0));
+
+ return protocol;
+}
+
+int test_server(const char *hostname)
+{
+ TLSContext context;
+ std::auto_ptr<SocketStreamTLS> conn = open_conn(hostname, context);
+ std::auto_ptr<BackupProtocolClient> apProtocol(
+ test_server_login(*conn));
+ BackupProtocolClient& protocol(*apProtocol);
+
// Make some test attributes
#define ATTR1_SIZE 245
#define ATTR2_SIZE 23
@@ -911,28 +948,11 @@
// BLOCK
{
- // Open a connection to the server
- SocketStreamTLS conn;
- conn.Open(context, Socket::TypeINET, hostname, BOX_PORT_BBSTORED);
-
- // Make a protocol
- BackupProtocolClient protocol(conn);
-
// Get it logging
FILE *protocolLog = ::fopen("testfiles/protocol.log", "w");
TEST_THAT(protocolLog != 0);
protocol.SetLogToFile(protocolLog);
- // Check the version
- std::auto_ptr<BackupProtocolClientVersion> serverVersion(protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION));
- TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION);
-
- // Login
- std::auto_ptr<BackupProtocolClientLoginConfirmed> loginConf(protocol.QueryLogin(0x01234567, 0));
-
- // Check marker is 0
- TEST_THAT(loginConf->GetClientStoreMarker() == 0);
-
#ifndef WIN32
// Check that we can't open a new connection which requests write permissions
{
@@ -1435,10 +1455,18 @@
}
//} skip:
+
+ std::auto_ptr<BackupStoreAccountDatabase> apAccounts(
+ BackupStoreAccountDatabase::Read(
+ "testfiles/accounts.txt"));
+ std::auto_ptr<BackupStoreRefCountDatabase> apRefCount(
+ BackupStoreRefCountDatabase::Load(
+ apAccounts->GetEntry(0x1234567), true));
// Create some nice recursive directories
int64_t dirtodelete = create_test_data_subdirs(protocol,
- BackupProtocolClientListDirectory::RootDirectory, "test_delete", 6 /* depth */);
+ BackupProtocolClientListDirectory::RootDirectory,
+ "test_delete", 6 /* depth */, *apRefCount);
// And delete them
{
@@ -1760,7 +1788,33 @@
TEST_THAT(TestDirExists("testfiles/0_2/backup/01234567"));
TEST_THAT(TestGetFileSize("testfiles/accounts.txt") > 8);
// make sure something is written to it
+
+ std::auto_ptr<BackupStoreAccountDatabase> apAccounts(
+ BackupStoreAccountDatabase::Read("testfiles/accounts.txt"));
+ std::auto_ptr<BackupStoreRefCountDatabase> apReferences(
+ BackupStoreRefCountDatabase::Load(
+ apAccounts->GetEntry(0x1234567), true));
+ TEST_EQUAL(BACKUPSTORE_ROOT_DIRECTORY_ID,
+ apReferences->GetLastObjectIDUsed());
+ TEST_EQUAL(1, apReferences->GetRefCount(BACKUPSTORE_ROOT_DIRECTORY_ID))
+ apReferences.reset();
+
+ // Delete the refcount database and log in again,
+ // check that it is recreated automatically but with
+ // no objects in it, to ensure seamless upgrade.
+ TEST_EQUAL(0, ::unlink("testfiles/0_0/backup/01234567/refcount.db.rfw"));
+
+ // Context
+ TLSContext context;
+ std::auto_ptr<SocketStreamTLS> conn = open_conn("localhost",
+ context);
+ test_server_login(*conn)->QueryFinished();
+
+ apReferences = BackupStoreRefCountDatabase::Load(
+ apAccounts->GetEntry(0x1234567), true);
+ TEST_EQUAL(0, apReferences->GetLastObjectIDUsed());
+
TEST_THAT(ServerIsAlive(pid));
TEST_THAT(test_server("localhost") == 0);