| cbk1994 |
11-27-2011 05:39 PM |
gsync: one-way folder sync from a Graal server to a web server
gsync is a simple tool for syncing folders from a Graal server to a web server running PHP (although you could implement it in any serverside language, if you wanted to).
Usage Scenarios
For example, I can use gsync to sync all script folders (weapons, scripts, npcs) to my remote server. I can then easily backup scripts with a single command. You can also backup binary files (e.g. images), so you could really sync your entire server if you wanted to.
It's very easy to combine this with some kind of post-sync operation like committing to a Mercurial or Git repository. This makes it very easy to keep track of changes and keep backups.
This script is probably better than GBall or the FTP server I wrote since it allows you to completely automate backups without issues the others have (GBall is slow and uses lots of memory, FTP is a pain to use and also somewhat slow).
How it Works
When running a sync, it will connect to the configured web server, authenticate with a key and IP matching, and then send a list of all files in the sync folders and their mod times. The web server then checks for files that have been deleted and deletes them from its local copy, and determines which files have been updated/added since the last sync, and tells the Graal server that it needs these files. The Graal servers sends these files, and the sync is complete.
Setup- Add func_http to your server.
- Decide on a key (ideally a long random string) and put this in the configuration section of both the PHP and the NPC scripts.
- Lookup your server's IP and put this in the PHP file.
- Upload the PHP file to some web server (tested on a linux server, but ought to work anywhere)
- Upload the database NPC to your server. Configure the stuff at the top.
- Make sure (npcserver) has read rights to anything you're planning on syncing.
- Call the sync function (suggestion: add it to your onRCChat commands).
If you don't want your files to be publicly accessible, you should be sure that your web server doesn't serve them (with the default configuration below, you can just prohibit access to the src folder).
Code
NPC GSync:
PHP Code:
enum LOG {
NONE,
ERROR,
INFO
};
function onInitialized() {
this.trigger("created");
}
function onCreated() {
this.join("func_http");
// options
this.server = "http://gsync.graalcenter.org/gsync.php"; // change this!
this.key = "PUT_SOMETHING_RANDOM_HERE";
this.verbosity = LOG.ERROR;
this.folders = {
"scripts",
"npcs",
"weapons"
};
}
public function sync() {
report(LOG.INFO, "Starting sync");
// build list of files and their mod times
report(LOG.INFO, "Building list of files and mod times");
temp.fileList = "";
for (temp.folder : this.folders) {
temp.files.loadFolder(temp.folder @ "/*", true);
for (temp.file : files) {
temp.path = temp.folder @ "/" @ temp.file;
temp.fileList @=
temp.path @ "\t" @ getFileModTime(temp.path) @ "\n";
}
}
report(LOG.INFO, "Built list of files and mod times");
// send list of files & mod times to the server
report(LOG.INFO, "Sending list of files and mod times to server");
temp.data = {
{"filelist", temp.fileList}
};
temp.returnData = this.send("filelist", temp.data);
if (temp.returnData == null) {
return report(LOG.ERROR, "Didn't receive acceptable response from server");
}
temp.neededFiles = temp.returnData.needed;
temp.returnData.destroy();
report(LOG.INFO, "Server responded with needed files (" @ temp.neededFiles.size() @ ")");
if (temp.neededFiles.size() > 0) {
// send server all the files it wants
temp.data = null;
temp.includedFiles = "";
for (temp.neededFile : temp.neededFiles) {
temp.contents.loadString(temp.neededFile);
temp.data.add({"file-" @ md5(temp.neededFile), temp.contents});
temp.data.add({"filem-" @ md5(temp.neededFile), getFileModTime(temp.neededFile)});
temp.includedFiles @= temp.neededFile @ "\n";
}
temp.data.add({"files", temp.includedFiles});
report(LOG.INFO, "Sending server requested files");
temp.returnData = this.send("store", temp.data);
if (temp.returnData == null) {
return report(LOG.ERROR, "Didn't receive acceptable response from server");
}
temp.returnData.destroy();
}
report(LOG.INFO, "Sync complete");
}
function send(temp.action, temp.data) {
temp.data.add({"action", temp.action});
temp.data.add({"key", this.key});
temp.json = new TStaticVar("GSyncResponse" @ int(timevar2 * 100));
temp.response = this.post(this.server, temp.data);
if (parseJSON(temp.json, temp.response)) {
report(LOG.INFO, "Parsed JSON successfully in server's response to " @ temp.action);
if (temp.json.status == "ok") {
return temp.json;
} else if (temp.json.status == "error") {
report(LOG.ERROR, "Received error from server for " @ temp.action @ ": " @ temp.json.error);
} else {
report(LOG.ERROR, "Received unexpected status code '" @ temp.json.status @ "' in response to " @ temp.action);
}
} else {
report(LOG.ERROR, "Unable to parse JSON in server's response to " @ temp.action);
}
temp.json.destroy();
}
function report(temp.logLevel, temp.msg) {
if (this.verbosity >= temp.logLevel) {
echo("gsync: " @ temp.msg);
}
}
gsync.php:
PHP Code:
<?php
$folder = "src"; // will be created if it doesn't exist
// make these match your Graal server
$ip = "50.23.136.183"; // find this on statistics.graal.us
$key = "PUT_SOMETHING_RANDOM HERE";
// don't touch below here
// make sure request is coming from a good client
error_reporting(0);
if ($_SERVER["REMOTE_ADDR"] != $ip || $_POST["key"] != $key) {
header("Status: 401 Unauthorized", 401);
$data = array(
"error" => "Bad IP or key"
);
sendResponse("error", $data);
die();
}
// client has successfully validated
// does the source folder exist?
if (! is_dir($folder)) {
mkdir($folder);
}
// check what command is being sent
switch ($_POST["action"]) {
case "filelist":
$fileList = $_POST["filelist"];
$fileList = explode("\n", $fileList);
$pathList = array(); // this will be filled with all paths to check for deleted files later
$neededList = array(); // this will be filled with all paths which need to be sent (e.g. updated files)
foreach ($fileList as $file) {
$parts = explode("\t", $file);
if (count($parts) != 2) {
continue;
}
$path = $parts[0];
$modTime = $parts[1];
$pathList[] = $path; // add to path list
// is the mod time from the server newer than the local copy?
$localModTime = filemtime($folder . "/" . $path);
if ($modTime > $localModTime) {
// the server has a newer version
$neededList[] = $path;
}
}
// do I need to delete any files?
$localPaths = getAllFiles($folder, array());
foreach ($localPaths as $localPath) {
if (! in_array($localPath, $pathList)) { // file has been deleted on server
unlink($folder . "/" . $localPath); // delete file from local copy
}
}
// tell the server which files, if any, I need
$data = array(
"needed" => $neededList
);
sendResponse("ok", $data);
break;
case "store":
$files = explode("\n", $_POST["files"]);
foreach ($files as $file) {
if (strlen($file) <= 0) {
continue;
}
// try to create folders
$path = substr($file, 0, strrpos($file, "/"));
mkdir($folder . "/" . $path, 0777, true);
$contents = $_POST["file-" . md5($file)];
$modTime = $_POST["filem-" . md5($file)];
file_put_contents($folder . "/" . $file, $contents);
touch($folder . "/" . $file, $modTime);
}
sendResponse("ok", array());
// you could easily add some post-sync script here, e.g. commit changes to a Mercurial repo
break;
default:
$data = array(
"error" => "Unrecognized action"
);
sendResponse("error", $data);
break;
}
function sendResponse($status, $data) {
$data["status"] = $status;
echo(json_encode($data));
}
// recursively get all files in a directory
function getAllFiles($dir, $files) {
global $folder;
$dh = opendir($dir);
while (($file = readdir($dh)) !== false) {
if (substr($file, 0, 1) == ".") {
continue; // starts with a dot (hidden or . or ..)
}
$fullPath = $dir . "/" . $file;
if (is_dir($fullPath)) {
$files = getAllFiles($fullPath, $files);
} else {
$files[] = substr($fullPath, strlen($folder) + 1);
}
}
return $files;
}
?>
If there are any issues/questions/suggestions I'll be happy to address them. |
