#!/usr/bin/env python
     # encoding: utf-8
     #
     # Copyright (c) 2009 Doug Hellmann All rights reserved.
     #
     """
     """
     #end_pymotw_header
     
     import robotparser
     import time
     import urlparse
     
     AGENT_NAME = 'PyMOTW'
     parser = robotparser.RobotFileParser()
     # Using the local copy
     parser.set_url('robots.txt')
     parser.read()
     parser.modified()
     
     PATHS = [
         '/',
         '/PyMOTW/',
         '/admin/',
         '/downloads/PyMOTW-1.92.tar.gz',
         ]
     
     for n, path in enumerate(PATHS):
         print
         age = int(time.time() - parser.mtime())
         print 'age:', age,
         if age > 1:
             print 're-reading robots.txt'
             parser.read()
             parser.modified()
         else:
             print
         print '%6s : %s' % (parser.can_fetch(AGENT_NAME, path), path)
         # Simulate a delay in processing
         time.sleep(1)