Merge pull request #82 from humanmade/add-robots.txt

Add basic support for robots.txt

server.js

@@ -2,6 +2,7 @@ var http   = require("http"),
 	url    = require("url"),
 	path   = require("path"),
 	fs     = require("fs"),
+	os     = require("os"),
 	tachyon= require( './index' ),
 	args = process.argv.slice(2),
 	port   = Number( args[0] ) ? args[0] : 8080,
@@ -32,6 +33,15 @@ http.createServer( function( request, response ) {
 		return response.end()
 	}
 
+	// robots.txt
+	if ( params.pathname === '/robots.txt' ) {
+		response.writeHead( 200, {
+			'Content-Type': 'text/plain'
+		} );
+		response.write( 'User-agent: *' + os.EOL + 'Allow: /' )
+		return response.end()
+	}
+
 	return tachyon.s3( config, decodeURIComponent( params.pathname.substr(1) ), params.query, function( err, data, info ) {
 		if ( err ) {
 			if ( debug ) {