zenodotus.repy

"""
<Program Name>
  zenodotus.repy

<Date Created>
  August 12, 2010

<Author(s)>
  Sebastian Morgan
  sebass63@gmail.com

<Major Edits>
  None

<Purpose>
  Using both the dnsserver and zenodotus_advertise modules, this program runs a 
  DNS name server to service hostname lookup requests within the Seattle 
  network.

<Notes>
  None

<Side Effects>
  Since this uses the dnsserver module to listen on some UDP port, it is 
  possible that this program might interfere with another application trying to 
  use the same port.

<Exceptions>
  MalformedPacketError
    _do_dns_callback will raise this if it encounters a dictionary which 
    describes a malformed packet.
  RefusedError
    _do_dns_callback will raise this if it encounters a dictionary that should 
    be refused for policy reasons.
  NotImplementedError
    _do_dns_callback will raise this if it encounters a dictionary requesting 
    a functionality which it does not provide.
  NonexistentDomainError
    _do_dns_callback will raise this if it encounters a dictionary requesting 
    information on a name with no valid corresponding advertise entry.
  ServerFailedError
    _do_dns_callback will raise this if it crashes and is not sure why.
"""



# This should be global, since zenodotus_advertise is interested in it.
_verbose = False

# Used to examine runtime resource use. Should not be modified more than once.
_ntp_progstart = 0




include dnsserver.repy
include zenodotus_advertise.repy
include ntp_time.repy

# This is the records we currently have cached. Should be iterated through 
# on a regular basis, removing expired entries. In addition, we should 
# decrement the entries; ttl values on each check.
record_cache = []

# This is the serial number which zenodotus will advertise in SOA records.
# Please increase this number by one each time a new version of the code is 
# pushed. NEVER decrease it.
_SOA_SERIAL = 6

# The file reference to which debug information should be saved. This is 
# initialized in the main code body.
_debug_stream = None

# This is initialized in the main body of the code, via getmyip(). This 
# must be reworked when commandline arguments are overhauled.
_host_ip_address = None




# Searches through the cache and returns all relevant records.
def _search_cache(record_name, record_type):
  answers = []

  for record in record_cache:
    if (record["name"] == record_name and record["type"] == record_type):
      answers.append(record)

  return answers



def _do_dns_callback(query_dictionary):
  """
  <Purpose>
    This method is invoked by dnsserver.repy as a callback whenever a 
    DNS-compliant query is received. The query_dictionary is a formatted 
    dictionary fully describing the DNS packet just received.

  <Arguments>
    A dictionary describing the DNS packet we've received. For format 
    specifics, please refer to dnsserver.repy.

  <Exceptions>
    There are five exceptions which can be raised by this method.
    They correspond to the DNS error codes, reproduced here.

    1 Format error - Packet was malformed
    2 Server failure - Server was unable to complete request
    3 Name Error - NXDOMAIN. Indicates that the name requested doesn't exist.
    4 Not Implemented - The server does not support the specified functionality.
    5 Refused - The server actively refused the connection.

    In order, the corresponding errors are as follows.

    MalformedPacketError
      Indicates that the dictionary we received contains a malformed packet.

    ServerFailedError
      Indicates that the server has failed in some capacity, but isn't able 
      to self-diagnose.

    NonexistentDomainError
      Indicates that the question asked by the user has no valid answer. More 
      specifically, that there is no corresponding value in advertise.

    NotImplementedError
      Indicates that the dictionary we received contains a question of a 
      form which we cannot answer due to lack of functionality.

    RefusedError
      Indicates that the dictionoary we received contains a packet which we 
      refused for policy reasons.

  <Side Effects>
    None

  <Returns>
    A dictionary with the same format as above, though with different values 
    associated with the keys. Additionally, 'answers' should be populated 
    with strings received from the advertise service.
  """
  # Prepare a new dictionary for responding.
  response_dictionary = {}

  try:
    # IQUERY and STATUS operations not yet implemented. (Worthwhile?)
    if query_dictionary['operation_code'] != 0:
      raise NotImplementedError()

    response_dictionary['questions'] = query_dictionary['questions']
    response_dictionary['query_response'] = True      # Always true
    response_dictionary['recursion_desired'] = query_dictionary['recursion_desired']
    response_dictionary['recursion_accepted'] = True  # We accept regardless
                                                      # of requests.
    response_dictionary['authority_advisory'] = False # Set to true later, 
                                                      # if valid.

    # The domain name service imposes an artificial 512 byte limit on all 
    # DNS UDP transmissions, meaning none of our messages can exceed that 
    # length. What's worse is that UDP headers are included in the length, 
    # and we have to service requests from IPv6 users as well. As such, we 
    # can only depend on having 512 - 128 (384) bytes available for any 
    # given message. If this limit is exceeded, the packet may be truncated 
    # at some point between server and client. At that time, this bit will 
    # be set to true.
    response_dictionary['truncation'] = False

    # Other entries we already know.
    response_dictionary['operation_code'] = 0
    response_dictionary['communication_id'] = query_dictionary['communication_id']
    response_dictionary['z'] = 0
    response_dictionary['error_code'] = 0
    response_dictionary['checking_disabled'] = query_dictionary['checking_disabled']
    response_dictionary['question_count'] = len(response_dictionary['questions'])
    response_dictionary['authentic_data'] = True

    answers = []

    # Iterate through the enclosed questions and respond appropriately.
    for question in query_dictionary['questions']:
      # Special variables for these, since we use them a lot.
      question_type = question['type']
      question_class = question['class']
      question_name = question['name']

      if question_class != 'IN':
        raise NotImplementedError("Non-IN query is unacceptable.")

      cached_values = _search_cache(question_name, question_type)
      if len(cached_values) > 0:
        for i in range(len(cached_values)):
          answers.append(cached_values[i])

      if len(answers) == 0: # We found nothing in the cache.
        string_answers = zenodotus_advertise_do_lookup(question_name, question_type)
        
    for entry in string_answers:
      if question_type == 'A':
        answers.append({'name': question_name,
                    'address': entry,
                    'type': question_type,
                    'time_to_live': 0,      # I still do not have a good idea 
                    'class': 'IN' })        # for this.

    response_dictionary['answers'] = answers
    response_dictionary['answer_count'] = len(answers)
    response_dictionary['additional_record_count'] = 0

    for record in answers:
      if record["type"] == "NS":
        response_dictionary['additional_record_count'] += 1

    response_dictionary['authority_record_count'] = len(answers) - response_dictionary['additional_record_count']

    return response_dictionary

  except NotImplementedError, e:
    if _verbose:
      print "NotImplementedError arose!"
      print "Dumping exception data . . ."
      print e
    _write_debug_report(_debug_stream, e, query_dictionary)
    raise NotImplementedError()
  except RefusedError, e:
    if _verbose:
      print "RefusedError arose!"
      print "Dumping exception data . . ."
      print e
    _write_debug_report(_debug_stream, "RefusedError", query_dictionary)
    raise RefusedError()
  except NonexistentDomainError, e:
    if _verbose:
      print "NonexistentDomainError arose!"
      print "Dumping exception data . . ."
      print e
    _write_debug_report(_debug_stream, "NXDomainError", query_dictionary)
    raise NonexistentDomainError()
  except MalformedPacketError, e:
    if _verbose:
      print "MalformedPacketError arose!"
      print "Dumping exception data . . ."
      print e
    _write_debug_report(_debug_stream, "MalformedPacketError", query_dictionary)
    raise MalformedPacketError()
  except Exception, e:
    if _verbose:
      print "Unknown error arose!"
      print "Dumping exception data . . ."
      print e
    _write_debug_report(_debug_stream, e, query_dictionary)
    raise ServerFailedError()




def _write_debug_report(debug_stream, error_data, packet_dictionary):
  """
  <Purpose>
    This method writes an error report to the specified file. These 
    reports are intended to be legible upon opening the file.

  <Arguments>
    debug_stream
      The file to which the debug report should be written.

    error_data
      A string describing the error that occurred. This can be 
      an exception string, or a custom message.

    packet_dictionary
      The dictionary produced by the packet which caused a problem. 
      This is included to shed light on the exception message in 
      the debug report.

  <Exceptions>
    IOError
      Occurs if disk is out of space.
    ValueError
      Occurs if file is closed.

  <Side Effects>
    It may be possible for a malicious entity to send deliberately malformed 
    packets to trigger this, depending on how it is used, and in so doing 
    fill the server hard drive in a relatively short time. Some sort of 
    protection against this is in order, but I'm not sure what the best way 
    to do that is.

  <Returns>
    None
  """
  print >> debug_stream, "=================================================="
  print >> debug_stream, "|         Error: Query caused a problem          |"
  print >> debug_stream, "=================================================="
  print >> debug_stream, "Exception data: \n"
  print >> debug_stream, error_data
  print >> debug_stream, "\n=================================================="
  print >> debug_stream, "Time processed: " + str(time_gettime()) + " NTP"
  print >> debug_stream, "=================================================="
  print >> debug_stream, "Dictionary State: \n"
  
  # This is some extremely complicated formatting, and I really wish there 
  # were a more convenient way to go about it.

  # In order to convert from decimal to hex properly, we need a character 
  # dictionary for quick reference.
  conversion_table = { 0: '0', 1: '1', 2: '2', 3: '3', 4: '4', 5: '5', 
                       6: '6', 7: '7', 8: '8', 9: '9', 10: 'A', 11: 'B', 
                       12: 'C', 13: 'D', 14: 'E', 15: 'F' }

  # We should make it simple to determine the max number of hex entries 
  # per line, in case this is refactored in the future.
  max_entries = 4

  # We also want to separate the parsing and formatting operations, so 
  # let's create a temporary array to store our hex data in.
  hex_values = []

  for entry in packet_dictionary:
    # When we know that we've hit the entry that can't be formatted in a 
    # simple way, we need to get tricky. We can't print as we go, because 
    # our printing will place a newline character at the end of every 
    # operation, which isn't what we want. So, we construct a string and 
    # populate it as we iterate.
    if entry == 'raw_data':
      output_text = "raw_data:\n  "

      # Grab the raw packet data:
      raw_data = packet_dictionary['raw_data']

      # This is formatted such that each character has a 0 - 255 decimal 
      # value, but if we try to print it, we might get an unreadable debug 
      # file! So we need to convert the values one by one, and put them in 
      # to a form that is easier on the eyes.
      for character in raw_data:
        # The & operation looks like this in a more logical format:
        # raw_data & 11110000
        # meaning we're finding the value of the first (upper) four 
        # bits as if they were a int4. This should be the decimal 
        # value of the first hex digit.
        upper_value = (ord(character) & 240) / 2 ** 4
      
        # Next, we need the decimal value of the lower four bits.
        lower_value = ord(character) & 15

        # Now that we have the decimal values of our two hex digits, we 
        # can easily convert them using the table we constructed earlier.
        hex_values.append(conversion_table[upper_value] + conversion_table[lower_value])

      # Once the hex value list is populated, we need to format the 
      # data and print. First, we'll iterate through and append the 
      # hex values to our output string.
      newline_index = 0

      for value in hex_values:
        output_text += value
        newline_index += 1
        # If we've already printed the max number of values on this line, we 
        # should move to the next one. Two spaces before each hex value 
        # should make this a little easier to read via indentation.
        if newline_index == max_entries:
          output_text += '\n  '
          newline_index = 0
        # Otherwise, print a space to separate this value from the next.
        else:
          output_text += ' '
      
      # Finally, print the raw data.
      print >> debug_stream, output_text

    else:
      print >> debug_stream, entry + ": " + str(packet_dictionary[entry])

  print >> debug_stream, "\n=================================================="
  debug_stream.flush()




def _evaluate_shorthand_time(expression):
  """
  <Purpose>
    Helper method to parse time duration shorthand, i.e.

           f('1h') = 3600

    This will accept the following formats:

       xm, xh, xd, xw

    Minutes, hours, days, weeks respectively.

  <Arguments>
    expression
      String expression to be evaluated

  <Exceptions>
    

  <Side Effects>
    ValueErrors possible in the case of malformed input.

  <Returns>
    An integer number of seconds.
  """
  # Validate input
  if (type(expression) != type("")):
    raise ValueError("Argument <expression> must be a string.")
  if (len(expression) == 0):
    raise ValueError("Time expression must not be empty.")
  if (expression[:-1].isalpha()):
    raise ValueError("Time expression " + str(expression) + "improperly formatted.")
  
  conversiontable = {'m' : 60, 'h' : 3600, 'd' : 86400, 'w' : 604800 }

  return conversiontable[expression[-1:]] * int(expression[:-1])




def _parse_zonefile(zonefile, maxcache=100):
  """
  <Purpose>
    Parses the given zone file and returns records which should be 
    initially cached. There's nothing clever here, it just uses the 
    first records it reads and caches those.

  <Arguments>
    zonefile
      The handle of the file to be parsed.
    maxcache
      The maximum number of records to be cached after reading.

  <Exceptions>
    TODO: Populate

  <Side Effects>
    None

  <Returns>
    A list of records in dictionary form. These have the same format 
    as the dictionaries in the answers[] list in the packet dictionary 
    specification.
  """
  # Break the file in to individual lines.
  lines = zonefile.readlines()

  # If a semicolon is present, remove it and everything following.
  # While we're at it, remove all leading and trailing whitespace.
  for i in range(len(lines)):
    if ';' in lines[i]:
      lines[i] = lines[i][0:lines[i].index(';')] # Repy should have regex!
    lines[i] = lines[i].strip()

  # Validate formatting of the first line.
  args = _splitstrip(lines[0])
  if len(args) != 2:
    raise ValueError("Malformed zone file (line 1)")
  if args[0] != "$ORIGIN":
    raise ValueError("Malformed zone file (line 1)")

  # Accept naively that the $origin arument is correctly formatted, 
  # because we never seem to use it.
  origin = args[1]
  del lines[0]

  # Validate formatting of the second line.
  args = _splitstrip(lines[0])
  if len(args) != 2:
    raise ValueError("Malformed zone file (line 2)")
  if args[0] != "$TTL":
    raise ValueError("Malformed zone file (line 2)")
  
  ttl = _evaluate_shorthand_time(args[1])
  del lines[0]

  records = []

  # Iterate through the remainder of the file, populating the records 
  # list as we go.
  i = 0
  while i < len(lines):
    line = lines[i]

    # Create default dictionary
    record = {}
    record["class"] = "IN"
    # Later, line-item assignment of ttl should be possible. For now, default.
    record["time_to_live"] = ttl

    # Add specified information.
    args = _splitstrip(line)
    if '(' in args:
      while not ')' in args:
        i += 1
        if not i in range(len(lines)):
          raise Exception("Unexpected EOF when parsing zone file.")
        line = lines[i]
        args += _splitstrip(lines[i])
      args.remove('(')
      args.remove(')')

    # TODO: Format checking for these fields.
    record["name"] = args[0]
    record["type"] = args[1]
    
    if record["type"] == "SOA":
      record["mname"] = args[2]
      record["rname"] = args[3]
      record["serial"] = int(args[4])
      record["refresh"] = _evaluate_shorthand_time(args[5])
      record["retry"] = _evaluate_shorthand_time(args[6])
      record["expire"] = _evaluate_shorthand_time(args[7])
      record["minimum"] = _evaluate_shorthand_time(args[8])
    elif record["type"] == "A":
      if args[2] == "getmyip":
        record["address"] = getmyip()
      else:
        record["address"] = args[2]
    elif record["type"] == "NS":
      record["address"] = args[2]
    #TODO: Add other queries

    records.append(record)

    i += 1

  return records



# Helper function for parsing
def _splitstrip(expression):
  args = expression.split(' ')
  for arg in args:
    arg = arg.strip()
  while '' in args:
    args.remove('')

  return args





if callfunc == 'initialize':
  listen_port = 0
  listen_ip = ''
  ntp_port = 0
  ntp_port_set = False

  # If no NTP port is specified, zenodotus chooses the first available 
  # port within this range.
  ntp_default_range = range(49000, 49005)
  success = True

  # Iterate through the call arguments, looking for flags that we know.
  # Currently implemented flags are:
  # -p        : <hostport>
  # -ip       : <IP Address>
  # -tsp/-ntp : <NTP port to use>
  # -v        : <Verbose flag>
  #
  # In addition to flags, we also allow the user to simply pass one 
  # hostport as an argument. While this is deprecated, it has been 
  # permitted in the past, and we should continue to allow it.
  if len(callargs) == 1:
    # We should ensure that we are only accepting integer ports.
    try:
      listen_port = int(callargs[0])
      print "Warning: You are using a deprecated syntax to run zenodotus."
      print "  Usage: zenodotus.repy -p <hostport>"
    except ValueError:
      print "Unacceptable argument: hostport must be a number."
      print "  Usage: zenodotus.repy <hostport>"
      success = False
      
      listen_ip = getmyip()
  elif len(callargs) == 0:
    print "Unacceptable arguments: No arguments! Need at least a hostport."
    print "  Usage: zenodotus.repy -p <hostport>"
    success = False
  else:
    # Check for flags. We use this ugly method of iteration for easy access 
    # of data near our iteration.
    for i in range(len(callargs)):
      if callargs[i] == '-p':
        try:
          listen_port = int(callargs[i + 1])
          # Only advance the flag if this works correctly.
          i += 1
        except ValueError:
          print "Unacceptable argument: hostport must be a number."
          print "  Usage: zenodotus.repy -p <hostport>"
          success = False
        except IndexError:
          print "Unacceptable arguments: -p must be followed by a hostport."
          print "  Usage: zenodotus.repy -p <hostport>"
          success = False
      if callargs[i] == '-v':
        _verbose = True
        print "Verbose mode requested."
      # This does not check for valid host IPs. If you input a bad IP, 
      # the program will crash somehow. Due to the possibility of the 
      # host having a IPv6 address, I don't know what the best way to 
      # do this is. Perhaps add a sniffer method to prevent bad args?
      # It's probably fine for now, but should be added in the near future.
      if callargs[i] == '-ip':
        try:
          listen_ip = callargs[i + 1]
        except IndexError:
          print "Unacceptable arguments: -ip must be followed by a IP Address."
          print "  Usage: zenodotus.repy -ip <IP Address>"
          success = False
      if callargs[i] == '-tsp':
        try:
          ntp_port = int(callargs[i + 1])
          ntp_port_set = True
        except IndexError:
          print "Unacceptable arguments: port must be a number."
          print "  Usage: zenodotus.repy -tsp <port>"
          success = False
        except ValueError:
          print "Unacceptable arguments: -tsp must be followed by a port."
          print "  Usage: zenodotus.repy -tsp <port>"
          success = False

  if listen_port == 0:
    print "Unacceptable arguments: No hostport provided!"
    print "  Usage: zenodotus.repy -p <hostport>"
    success = False

  if not ntp_port_set:
    print "No NTP port specified. Trying defaults . . ."
    ntp_updated = False
    ntp_test_index = 0
    while not ntp_updated:
      try:
        print "Testing port " + str(ntp_default_range[ntp_test_index]) + " . . ."
        time_updatetime(ntp_default_range[ntp_test_index])
        time_data = time_gettime()
        ntp_updated = True
        print "  Success!"
      except Exception, e:
        print "  Port failed, with exception: " + str(e)
        if ntp_test_index + 1 < len(ntp_default_range):
          ntp_test_index += 1
        else:
          print "All defaults failed. Unable to set NTP port."
          print "Zenodotus will shut down."
          # Convention.
          success = False
          exitall()
    
  if listen_ip == '':
    listen_ip = getmyip()


  if success:
    # Open a file writer for debug output.
    _debug_stream = open('debug.txt', 'w')
    _host_ip_address = listen_ip

    # Update NTP Time on the server, so that we can use time_gettime() for 
    # timestamping purposes. This should incorporate a way to run zenodotus 
    # without timestamping eventually.
    if ntp_port_set:
      try:
        time_updatetime(ntp_port)
        _ntp_startprog = time_gettime()
        time_acquired = True
        print "\nTime acquired. Zenodotus started at " + str(_ntp_startprog) + " NTP."
      except TimeError:
        print "\nTime acquisition failed; NTP server could not be reached."
        print "Zenodotus will shut down."
        exitall()

    print "Attempting to load hosts.txt data . . . "
    file_hwnd = open("hosts.txt", "r")
    record_cache = _parse_zonefile(file_hwnd)
    print "Loaded successfully. Cached " + str(len(record_cache)) + " permanent records."

    dnsserver_registercallback(listen_port, listen_ip, _do_dns_callback)
    print "DNS callback registered."
    print "zenodotus now running on the local machine:"
    print "  Host port = " + str(listen_port)
    print "  Local IP  = " + str(listen_ip)