Add /robots.txt

httpbin/handlers.go

@@ -554,3 +554,11 @@ func (h *HTTPBin) Range(w http.ResponseWriter, r *http.Request) {
 func (h *HTTPBin) HTML(w http.ResponseWriter, r *http.Request) {
 	writeHTML(w, MustAsset("moby.html"), http.StatusOK)
 }
+
+// Robots renders a basic robots.txt file
+func (h *HTTPBin) Robots(w http.ResponseWriter, r *http.Request) {
+	robotsTxt := []byte(`User-agent: *
+Disallow: /deny
+`)
+	writeResponse(w, http.StatusOK, "text/plain", robotsTxt)
+}

httpbin/handlers_test.go

@@ -1543,3 +1543,12 @@ func TestHTML(t *testing.T) {
 	assertContentType(t, w, htmlContentType)
 	assertBodyContains(t, w, `<h1>Herman Melville - Moby-Dick</h1>`)
 }
+
+func TestRobots(t *testing.T) {
+	r, _ := http.NewRequest("GET", "/robots.txt", nil)
+	w := httptest.NewRecorder()
+	handler.ServeHTTP(w, r)
+
+	assertContentType(t, w, "text/plain")
+	assertBodyContains(t, w, `Disallow: /deny`)
+}

httpbin/httpbin.go

@@ -123,6 +123,7 @@ func (h *HTTPBin) Handler() http.Handler {
 	mux.HandleFunc("/range/", h.Range)
 
 	mux.HandleFunc("/html", h.HTML)
+	mux.HandleFunc("/robots.txt", h.Robots)
 
 	// Make sure our ServeMux doesn't "helpfully" redirect these invalid
 	// endpoints by adding a trailing slash. See the ServeMux docs for more