#!/bin/bash
curl -L -o $1.robots.txt https://$1/robots.txt