"""
Zendesk XML data import

This script will import a tar/gzipped file generated by Zendesk into Askbot.
For more info see https://support.zendesk.com/entries/23002207 (use XML 
instead of CSV)

The tgz archive must contain a single directory with the following xml 
files:
    accounts.xml (ignored)
    categories.xml (ignored)
    entries.xml
    forums.xml
    groups.xml (ignored)
    organizations.xml
    posts.xml
    tickets.xml
    users.xml

You have the ability to filter choose whether to import forums, tickets, or
both. Additionally, you can specify whether you wish to filter the content
further by specific forums, tags, and date.

FORUMS:
    Importing forums will give you a list of your public forums to choose from.
    Private forums will not be shown by default. Once you choose the forums 
    you wish to import, you can filter down the entries by tags (any entry
    with any matching tag will be imported), and by date range (of created_at
    datetime).

    Votes on Entries will be converted to Votes for the question in Askbot.
    View counts are transferred as well. If a Post is marked as is_informative,
    it will mark the answer as accepted. Note that since Zendesk supports 
    multiple "accepted" answers and Askbot does not, this will cause the 
    accepted answer in Askbot to be the most recent accepted Post.

TICKETS:
    Importing Tickets will give you an option to filter down the entries by 
    tags (any Ticket with any matching tag will be imported), and by date range 
    (of created_at datetime). 

    Tickets don't have any view count or vote stats so none of that info is 
    transferred. Additionally, there's no easy way to determine which comment
    on the Ticket may be the accepted answer so the script doesn't mark any
    answer as accepted.

    Private comments are not imported.

REQUIREMENTS:
    This script requires the lxml module which is not part of the base
    Askbot install. The lxml module will require your server have the
    libxml2-devel and libxslt-devel packages installed in order to 
    install correctly.

NOTES:
    Running this import will truncate the existing zendesk_* tables in order
    to ensure you don't end up re-importing existing data.

    If your site is configured to only allow a single answer per user
    (LIMIT_ONE_ANSWER_PER_USER = True), you will be prompted to disable this 
    setting temporarily while the import proceeds. It will turn it back on 
    when complete. If you choose not to disable this setting, then the import 
    will add any additional answers from a user as comments on their original 
    answer. This is not ideal so it's encouraged you agree to turn this 
    setting off while doing the import. 

Run this command as::
    python manage.py import_zendesk /path/to/zendesk/archive.tgz

TODO: 
    - Use logging for more verbose output
    - Add option to import Attachments from existing Zendesk installation
"""
import os
import re
import sys
import tarfile
import tempfile
from datetime import datetime, date
from lxml import etree
from django.core.management.base import BaseCommand, CommandError
from django.conf import settings
from django.db import transaction
from django.db import connection
from askbot import models as askbot_models
from askbot.utils import console
from askbot.utils.html import unescape
from askbot import exceptions as askbot_exceptions
from askbot.conf import settings as askbot_settings
from askbot.importers.zendesk import models as zendesk_models

# a hack, did not know how to parse timezone offset
# todo: clean this up
ZERO_TIME = datetime.strptime('00:00', '%H:%M')
# load admin user where a user is needed (eg. user who closed thread)
ADMIN_USER = askbot_models.User.objects.filter(is_superuser=True)[:1]
# option choices for what data to import from Zendesk
DATA_IMPORT_ALL = 0
DATA_IMPORT_FORUMS = 1
DATA_IMPORT_TICKETS = 2

# used for seeding Vote count when importing Zendesk Forum content
try:
    PHANTOM_VOTER_USER = askbot_models.User.objects.get(username='phantom_voter')
except askbot_models.User.DoesNotExist:
    PHANTOM_VOTER_USER = askbot_models.User(
            username = 'phantom_voter',
            first_name = 'Phantom',
            last_name = 'Voter',
            real_name = 'Phantom Voter',
            date_joined = datetime.now(),
            is_active = False,
            about = 'Fake account for seeding vote counts during Zendesk import',
        ).save()

def ensure_unique_username(name_seed):
    """Returns unique user name, by modifying the name if the same name exists 
    in the database until the modified name is unique.

    :param name_seed: (str) proposed user name

    :returns: (str) validated unique user name
    """
    original_name = name_seed
    attempt_no = 1
    while True:
        try:
            askbot_models.User.objects.get(username = name_seed)
            name_seed = original_name[:29] + str(attempt_no)
            attempt_no += 1
        except askbot_models.User.DoesNotExist:
            return name_seed

def create_askbot_user(zd_user):
    """Create askbot user from Zendesk User record

    Zendesk User fields that are copied over or otherwise translated:
        name
        username
        email
        Organization name matching organization_id
        is_verified
        is_active
        last_login
        created_at

    :param zd_user: (obj) zendesk_models.User object to create Askbot user
    from.

    :returns: (mixed) askbot user object or None if there is an error
    """
    if zd_user.email is None:
        username = zd_user.name.replace(" ", "_").lower()
        email = ''
    else:
        username = zd_user.email
        email = zd_user.email
    username = ensure_unique_username(username[:30])

    # last_seen cannot be null
    last_seen = zd_user.last_login
    if not last_seen:
        last_seen = zd_user.created_at

    # lookup organization name (todo: cache this)
    about = ""
    if zd_user.organization_id:
        try:
            org = zendesk_models.Organization.objects.get(organization_id=zd_user.organization_id)
            about = org.name
        except zendesk_models.Organization.DoesNotExist:
            pass

    ab_user = askbot_models.User(
        username = username,
        first_name = zd_user.name.rpartition(' ')[0].strip()[:30],
        last_name = zd_user.name.rpartition(' ')[2].strip()[:30],
        real_name = zd_user.name[:100],
        email = email,
        email_isvalid = zd_user.is_verified,
        date_joined = zd_user.created_at,
        last_seen = last_seen,
        is_active = zd_user.is_active,
        about = about,
    )
    ab_user.save()
    return ab_user

def seed_post_with_votes(post, votes_count):
    """Seed imported Question with an initial vote count

    Votes are set in multple locations for caching. points = (vote_up_count - 
    vote_down_count). Since we're creating the post now and Zendesk doesn't 
    have down votes, we just calculate this as up votes. 

    Vote objects require a user. We have created an inactive PHANTOM_VOTER_USER
    above to artificially serve this purpose. 
    
    NOTE: Vote objects are indended to be +1 (VOTE_UP) or -1 (VOTE_DOWN). 
    We're overriding this by adding a Vote object that is +votes_count to 
    create a weighted artificial Vote. This may cause problems if the votes
    are recalculated for some reason later.

    :param post: (obj) the askbot.models.Post object to seed with the votes
    :param votes_count: (int) number of votes to seed the Post with
    """
    post.points = votes_count
    post.vote_up_count = votes_count
    post.save()
    post.thread.points = votes_count
    post.thread.save()
    askbot_models.Vote(user=PHANTOM_VOTER_USER, voted_post=post, 
                       vote=votes_count, voted_at=datetime.now()).save()

def post_question(zendesk_entry):
    """Posts question to askbot from Zendesk Entry

    Translates Zendesk Entry to an Askbot question. Links correct user,
    updates the view count and vote count. Closes the question if the 
    Entry is locked. 

    :param zendesk_entry: (obj) zendesk_models.Entry object

    :returns: (obj) askbot Post object if it succeeded. None if there was
    an error.
    """
    try:
        askbot_post = zendesk_entry.get_author().post_question(
            title = zendesk_entry.title,
            body_text = zendesk_entry.get_body_text(),
            tags = zendesk_entry.get_tag_names(),
            timestamp = zendesk_entry.created_at,
        )
        # seed the views with the # hits we had on zendesk
        askbot_post.thread.increase_view_count(increment=zendesk_entry.hits)
        if zendesk_entry.votes_count:
            seed_post_with_votes(askbot_post, zendesk_entry.votes_count)

        # close threads that were locked in Zendesk and assign a default
        # reason of "question answered". Set default user to admin.
        if zendesk_entry.is_locked:
            askbot_post.thread.set_closed_status(
                closed=True, 
                closed_by=ADMIN_USER, 
                closed_at=datetime.now(), 
                close_reason=5)
            askbot_post.thread.save()
        return askbot_post
    except Exception, e:
        msg = unicode(e)
        print "Warning: entry %d skipped: %s" % (zendesk_entry.entry_id, msg)

def post_question_from_ticket(zendesk_ticket):
    """Posts question to Askbot from Zendesk Ticket

    Translates Zendesk Ticket to an Askbot question. View count and votes 
    aren't relevant on Tickets in Zendesk so we don't seed any of that info
    (like we do on post_question()). 

    :param zendesk_ticket: (obj) zendesk_models.Ticket object

    :returns: (obj) askbot Post object if it succeeded. None if there was
    an error.

    :todo: wrap this into post_question()
    """
    try:
        askbot_post = zendesk_ticket.get_author().post_question(
            title = zendesk_ticket.subject,
            body_text = zendesk_ticket.get_body_text(),
            tags = zendesk_ticket.get_tag_names(),
            timestamp = zendesk_ticket.created_at
        )
        return askbot_post
    except Exception, e:
        msg = unicode(e)
        print "Warning: ticket %d skipped: %s" % (zendesk_ticket.ticket_id, msg)

def post_comment(source_post, parent):
    """Post comment on an answer from a Zendesk Post or Comment.

    :param source_post: (obj) A zendesk_models.Post or zendesk_models.Comment
    object
    :param parent: (obj) Askbot Post object which will be the parent of the 
    comment

    :returns: (obj) Askbot Post object with post_type='comment' or None if
    there was an error.
    """
    try:
        askbot_comment = source_post.get_author().post_comment(
            parent_post = parent,
            body_text = source_post.get_body_text(),
            timestamp = source_post.created_at
        )
        return askbot_comment
    except Exception, e:
        msg = unicode(e)
        print "Warning: post %d skipped: %s" % (zendesk_post.post_id, msg)

def post_answer(zendesk_post, question):
    """Posts an answer to Askbot, from a Zendesk Post

    If the Post was marked as informative in Zendesk, we mark it as an accepted
    answer in Askbot. Since Askbot only allows a single accepted answer and 
    Zendesk supports multiple answers, this will re-mark the answer
    for each one and ultimately end on the most recent post. This may not be
    the most relevant answer in the end.

    If Askbot is configured to only allow a single answer per user, any
    additional answers from the user will be added as comments on the original
    answer from the user. This will likely create some context confusion so
    it's recommended you have this setting during for the import.

    :param zendesk_post: (obj) zendesk_models.Post object to create answer from
    :param question: (obj) Askbot Post object with post_type='question' to post
    the answer to.

    :returns: (obj) Askbot Post object with post_type='answer' or 'comment'
    depending on the setting for LIMIT_ONE_ANSWER_PER_USER or None if
    there was an error.
    """
    try:
        askbot_post = zendesk_post.get_author().post_answer(
            question = question,
            body_text = zendesk_post.get_body_text(),
            timestamp = zendesk_post.created_at
        )
        if zendesk_post.is_informative:
            askbot_post.thread.accepted_answer_id = askbot_post.id
            askbot_post.thread.save()
        return askbot_post
    except askbot_exceptions.AnswerAlreadyGiven:
        answer = question.thread.get_answers_by_user(user=zendesk_post.get_author())[0]
        askbot_comment = post_comment(zendesk_post, answer)
        return askbot_comment
    except Exception, e:
        msg = unicode(e)
        print "Warning: post %d skipped: %s" % (zendesk_post.post_id, msg)

def post_answer_from_comment(zendesk_comment, question):
    """Posts an answer to Askbot, from Zendesk Comment on a ticket

    If Askbot is configured to only allow a single answer per user, any
    additional answers from the user will be added as comments on the original
    answer from the user. This will likely create some context confusion so
    it's recommended you have this setting during for the import.

    There is no reliable way to know which comment is the "accepted" answer
    so we don't try and set that automatically.

    :param zendesk_comment: (obj) zendesk_models.Comment object to create 
    answer from.
    :param question: (obj) Askbot Post object with post_type='question' to post
    the answer to.   
    """
    if not zendesk_comment.is_public:
        return
    try:
        askbot_post = zendesk_comment.get_author().post_answer(
            question = question,
            body_text = zendesk_comment.get_body_text(),
            timestamp = zendesk_comment.created_at
        )
        return askbot_post
    except askbot_exceptions.AnswerAlreadyGiven:
        answer = question.thread.get_answers_by_user(user=zendesk_comment.get_author())[0]
        askbot_comment = post_comment(zendesk_comment, answer)
        return askbot_comment
    except Exception, e:
        msg = unicode(e)
        print "Warning: comment %d skipped: %s" % (zendesk_comment.id, msg)

def get_xml_element_val(elem, field_name):
    """Return the value of the etree element for field_name and cast it to the
    correct data type.

    :param elem: (obj) etree element object to search
    :param field_name: (str) field name to search for in elem

    :returns: (mixed) value of field_name in etree object cast to the correct
    native Python data type
    """
    field = elem.find(field_name)
    if field is None:
        return None
    try:
        field_type = field.attrib['type']
    except KeyError:
        field_type = ''
    raw_val = field.text
    if raw_val is None:
        return None
    if field_type == 'boolean':
        if raw_val == 'true':
            return True
        elif raw_val == 'false':
            return False
        else:
            raise ValueError('"true" or "false" expected, found "%s"' % raw_val)
    elif field_type.endswith('integer'):
        return int(raw_val)
    elif field_type == 'datetime':
        if raw_val is None:
            return None
        # todo: clean this up
        raw_datetime = raw_val[:19]
        tzoffset_sign = raw_val[19]
        raw_tzoffset = raw_val[20:]
        if raw_val:
            dt = datetime.strptime(raw_datetime, '%Y-%m-%dT%H:%M:%S')
            tzoffset_amt = datetime.strptime(raw_tzoffset, '%H:%M')
            tzoffset = tzoffset_amt - ZERO_TIME
            if tzoffset_sign == '-':
                return dt - tzoffset
            else:
                return dt + tzoffset
        else:
            return None
    elif field_type == 'array':
        # returns a list of child elements
        # comments > comment
        sfield_name = field_name[:-1]
        return field.findall(sfield_name)
    else:
        return raw_val

def toggle_user_answer_limit_setting(val):
    """Turns the Askbot live_setting for LIMIT_ONE_ANSWER_PER_USER on 
    or off.

    :param val: (bool) value to set LIMIT_ONE_ANSWER_PER_USER to
    """
    if val:
        askbot_settings.update('LIMIT_ONE_ANSWER_PER_USER', True)
    else:
        askbot_settings.update('LIMIT_ONE_ANSWER_PER_USER', False)
    print "set LIMIT_ONE_ANSWER_PER_USER to %s" % val

def check_user_answer_limit():
    """Checks if LIMIT_ONE_ANSWER_PER_USER is True, if so, warn the user
    and give them an option to turn it off temporarily for the import.

    The import is really messy if we don't allow multiple answers for a user
    when translating from Zendesk to Askbot. If the user opts to turn this 
    off at the beginning of the import it will be turned on automatically when 
    we're done.
    """
    if not askbot_settings.LIMIT_ONE_ANSWER_PER_USER:
        return
    else:
        print 
        print "*"*64
        print "* WARNING"
        print "*"*64
        print "* Your settings are currently limiting users to a single"
        print "* answer per question. Zendesk doesn't translate well to"
        print "* this. It's highly recommended you let us switch this"
        print "* off temporarily while the import proceeds. We'll switch"
        print "* it back on when we're done."
        print "*"
        print "* If you choose not to do this, each additional post on"
        print "* a forum topic or additional comment on a ticket will be"
        print "* appended as a comment on to the first answer by the user."
        print "*" * 64
        prompt = "Okay to turn off the LIMIT_ONE_ANSWER_PER_USER setting?"
        response = console.get_yes_or_no(prompt, 'yes')
        if response == 'yes':
            toggle_user_answer_limit_setting(False)
            print
            return True
        print

class Command(BaseCommand):
    def handle(self, *args, **kwargs):
        """Base handler for command run from command line
        
        Walks the user through the complete import process. 

        Checks the LIMIT_ONE_ANSWER_PER_USER and prompts the user to turn it 
        OFF if it is currently enabled since imported data will make more sense
        with it off temporarily. If it is on prior to the import and they 
        agree to turn it off, the importer will turn it back on when the import
        is complete.

        Prompts user to choose whether they would like to import Forums & 
        Tickets, Forums only, or Tickets only. Users are automatically imported
        as they are required to link new Posts to. Organizations are only used
        for looking up the organization name to add the user's profile.

        All content of the required xml files is loaded into separate Zendesk
        model tables as an intermediate step. 

        When importing Forums, the user is given the choice to choose which
        Forums they would like to import content from. 

        Tag Filters
        The user is able to specify tags to filter on (importing any Forum or
        Ticket entries matching ANY of the tags specified). 

        Date Filters
        The user is able to specify a date range for filtering Forum Posts by
        date created.  

        These options allow some simple control over potentially large amounts
        of data, weeding out noise and otherwise outdated or irrelevant
        content.

        :param args: (mixed) positional arguments to command. We require
        a single str argument here as the full path to the Zendesk tgz archive
        containing the xml files.
        """
        if len(args) != 1:
            raise CommandError('Please provide the path to the Zendesk tgz archive.')

        self.tar = tarfile.open(args[0], 'r:gz')

        # ask what data we are importing
        print
        print
        print "-"*64
        print "This script will attempt to import your Zendesk data into"
        print "Askbot. If you are importing into an existing installation,"
        print "** backup your database before continuing **!"
        print "-"*64
        print "You will have a chance to decide if you want to import"
        print "tickets, forums, or both. Additional options are presented"
        print "to filter the imported content by forum, tag, and date."
        print "Users are always imported."
        print "-"*64

        user_answer_limit_reset = check_user_answer_limit()
        choices = ['Forums and Tickets', 'Forums Only', 'Tickets Only']
        prompt = "What data do you wish to import from Zendesk?"
        data_choice = console.numeric_choice_dialog(prompt, choices=choices)
        print

        # read relevant data into temporary tables. We read everything and then
        # filter when we actually import into the Askbot tablespace
        sys.stdout.write("Reading organizations.xml... ")
        self.read_organizations()
        sys.stdout.write("Reading users.xml... ")
        self.read_users()
        if data_choice in [DATA_IMPORT_ALL, DATA_IMPORT_FORUMS]:
            sys.stdout.write("Reading forums.xml... ")
            self.read_forums()
            sys.stdout.write("Reading entries.xml... ")
            self.read_entries()
            sys.stdout.write("Reading posts.xml... ")
            self.read_posts()
        if data_choice in [DATA_IMPORT_ALL, DATA_IMPORT_TICKETS]:
            sys.stdout.write("Reading tickets.xml... ")
            self.read_tickets()

        # forums choices
        # ---------------------------------------------------------------------
        print 
        if data_choice in [DATA_IMPORT_ALL, DATA_IMPORT_FORUMS]:
            print "="*64
            print " FORUMS"
            print "="*64
            forum_choices = self.prompt_for_forums()
            forum_tag_choices = self.prompt_for_tags()
            (forum_date_filter) = self.prompt_for_date()

        # tickets choices
        # ---------------------------------------------------------------------
        if data_choice in [DATA_IMPORT_ALL, DATA_IMPORT_TICKETS]:
            print
            print "="*64
            print " TICKETS"
            print "="*64
            ticket_tag_choices = self.prompt_for_tags()
            (ticket_date_filter) = self.prompt_for_date()

        # import data
        # ---------------------------------------------------------------------
        sys.stdout.write("Importing user accounts: ")
        self.import_users()
        if data_choice in [DATA_IMPORT_ALL, DATA_IMPORT_FORUMS]:        
            self.import_forums(forum_choices, forum_tag_choices, forum_date_filter)
        if data_choice in [DATA_IMPORT_ALL, DATA_IMPORT_TICKETS]:
            self.import_tickets(tags=ticket_tag_choices, date_filter=ticket_date_filter)

        # cleaning up
        # ---------------------------------------------------------------------
        if user_answer_limit_reset:
            toggle_user_answer_limit_setting(True)
        print
        print "Done!"
        print

    
    def prompt_for_forums(self):
        """Prompt user to select the forums they'd like to import or choose all
        of them

        :returns: (list) zendesk_models.Forum objects selected by user
        """
        # special case for console.numeric_multiple_choice_dialog 0 = all
        ALL_FORUMS = 0
        public_forums = zendesk_models.Forum.objects.filter(is_public='t').order_by('forum_id')
        choices = [f.name for f in public_forums]
        prompt = "Which forums do you want to import (separate multiple choices by a space)?"
        numeric_choices = console.numeric_multiple_choice_dialog(prompt, choices=choices, all_option=True)
        if ALL_FORUMS in numeric_choices:
            return public_forums
        return [public_forums[f-1] for f in numeric_choices]

    def prompt_for_tags(self):
        """Prompt user for a space-separated list of tags to filter imported 
        objects by.

        Tags are case-insensitive for the import and everything is forced to 
        lowercase.

        :returns: (list) tags specified by user
        """
        prompt = "Enter tags separated by spaces to filter by (leave blank for all):"
        tags = console.simple_dialog(prompt, required=False)
        return tags.split()

    def prompt_for_date(self):
        """Prompt user for start and end dates in YYYY-MM-DD format for
        filtering imported objects by a date range of when they were created.

        :returns: (tuple) 2 datetime objects representing the start date and 
        end date respectively to filter by. Either or both of the elements can
        also be None indicating no filter is required for that boundary. 
        """
        start_date = False
        end_date = False
        while not start_date:
            prompt = "Enter earliest date (yyyy-mm-dd) to import content from (leave blank for all):"
            start = console.simple_dialog(prompt, required=False).strip()
            if start:
                try:
                    start_date = datetime.strptime(start,"%Y-%m-%d")
                except ValueError:
                    print
                    print "*** Please enter a date in the format YYYY-MM-DD or leave it blank ***"
            else:
                start_date = None
                break
        while not end_date:
            prompt = "Enter latest date (yyyy-mm-dd) to import content from (leave blank for all):"
            end = console.simple_dialog(prompt, required=False).strip()
            if end:
                try:
                    end_date = datetime.strptime(end,"%Y-%m-%d")
                except ValueError:
                    print
                    print "*** Please enter a date in the format YYYY-MM-DD or leave it blank ***"
            else:
                end_date = None
                break
        return (start_date, end_date)

    def get_file(self, file_name):
        """Opens file and reads in xml data

        :param file_name: (str) full path to Zendesk export tgz file

        :returns: (obj) etree object for traversing xml element tree
        """
        first_item = self.tar.getnames()[0]
        file_path = file_name
        if not first_item.endswith('.xml'):
            file_path = os.path.join(first_item, file_path)
            
        file_info = self.tar.getmember(file_path)
        xml_file = self.tar.extractfile(file_info)
        return etree.parse(xml_file)

    @transaction.autocommit
    def read_xml_file(self,
            file_name = None,
            entry_name = None,
            model = None,
            fields = None,
            extra_field_mappings = None,
            sub_entities = []
        ):
        """Reads xml file, parses entries into Zendesk model objects, and saves
        them to the database. 

        Values are cast to their correct data types. 

        Sub-entities are used for extracting an embedded structure from the 
        element tree into a separate model and table.

        :param file_name: (str) name of xml file,
        :param entry_name: (str) name of entries to read from the xml file
        :param model: (obj) model where data will be stored
        :param fields: (list) field names (str) in xml that will be translated 
        to model fields by simple substitiution of '_' for '-'
        :param extra_field_mappings (tuple) list of two tuples for xml field 
        names have specific translation that doesn't follow the standard for 
        the fields parameter
        :param sub_entities: (list) of dicts describing fields that should be 
        treated as separate models (like Ticket.comments). The structure is
        similar to this method. Each dict key is the field name to be treated
        as a sub-entity. The value is a tuple with (model, [sub-entity fields],
        (sub-entity extra_field_mappings)).
                    [{'comments': (
                        zendesk_models.Comment, 
                        ['author-id', 'created-at', 'is-public', 'type', 
                            'value', 'via-id', 'ticket-id'], 
                        (),)
                    }]
        todo: support blank values vs. nulls for strings
        """
        cursor = connection.cursor()
        cursor.execute('TRUNCATE TABLE "{0}" CASCADE'.format(model._meta.db_table))
        xml = self.get_file(file_name)
        items_saved = 0
        for xml_entry in xml.findall(entry_name):
            instance = model()
            for field in fields:
                value = get_xml_element_val(xml_entry, field)
                model_field_name = field.replace('-', '_')
                max_length = instance._meta.get_field(model_field_name).max_length
                if value and max_length:
                    value = value[:max_length]
                setattr(instance, model_field_name, value)
            if extra_field_mappings:
                for (field, model_field_name) in extra_field_mappings:
                    value = get_xml_element_val(xml_entry, field)
                    setattr(instance, model_field_name, value)

            sub_instances = []
            for sub_entity in sub_entities:
                for sub_field_name, sub_def in sub_entity.iteritems():
                    sub_list = get_xml_element_val(xml_entry, sub_field_name)
                    sub_model, sub_fields, sub_extra_field_mappings = sub_def
                    for child in sub_list:
                        sub_instance = sub_model()
                        for sub_field in sub_fields:
                            sub_value = get_xml_element_val(child, sub_field)
                            sub_model_field_name = sub_field.replace('-', '_')
                            sub_max_length = sub_instance._meta.get_field(sub_model_field_name).max_length
                            if sub_value and sub_max_length:
                                sub_value = sub_value[:sub_max_length]
                            setattr(sub_instance, sub_model_field_name, sub_value)
                        sub_instances.append(sub_instance)

            instance.save()
            for si in sub_instances:
                # set the parent id
                setattr(si, "%s_id" % entry_name, instance.id)
                si.save()
            items_saved += 1
            console.print_action('%d' % items_saved)
        console.print_action('%d total' % items_saved, nowipe = True)


    def read_organizations(self):
        """Read Zendesk Organizations from xml file and save them as Zendesk 
        models
        """
        self.read_xml_file(
            file_name = 'organizations.xml',
            entry_name = 'organization',
            model = zendesk_models.Organization,
            fields = (
                'created-at', 'default', 'details', 'external-id', 'group-id', 
                'is-shared', 'is-shared-comments', 'name', 
                'notes', 'suspended', 'updated-at'
            ),
            extra_field_mappings = (('id', 'organization_id'),)
        )

    def read_users(self):
        """Read Zendesk Users from xml file and save them as Zendesk 
        models
        """
        self.read_xml_file(
            file_name = 'users.xml',
            entry_name = 'user',
            model = zendesk_models.User,
            fields = (
                'created-at', 'is-active', 'last-login', 'name',
                'openid-url', 'organization-id', 'phone', 'restriction-id',
                'roles', 'time-zone', 'updated-at', 'uses-12-hour-clock',
                'email', 'is-verified', 'photo-url'
            ),
            extra_field_mappings = (('id', 'zendesk_user_id'),)
        )

    def read_entries(self):
        """Read Zendesk Entries from xml file and save them as Zendesk 
        models. 

        Entries in Zendesk are top-level posts in a forum.
        """
        self.read_xml_file(
            file_name = 'entries.xml',
            entry_name = 'entry',
            model = zendesk_models.Entry,
            fields = (
                'body', 'created-at', 'flag-type-id', 'forum-id',
                'hits', 'entry-id', 'is-highlighted', 'is-locked', 'is-pinned',
                'is-public', 'organization-id', 'position', 'posts-count', 
                'submitter-id', 'title', 'updated-at', 'votes-count'
            ),
            extra_field_mappings = (
                ('id', 'entry_id'),
                ('current-tags', 'tags'),
            )
        )

    def read_posts(self):
        """Read Zendesk Posts from xml file and save them as Zendesk 
        models.

        Posts in Zendesk are children of Entries. They are like replies
        on a top-level forum post.
        """
        self.read_xml_file(
            file_name = 'posts.xml',
            entry_name = 'post',
            model = zendesk_models.Post,
            fields = (
                'body', 'created-at', 'updated-at', 'entry-id',
                'forum-id', 'user-id', 'is-informative'
            ),
            extra_field_mappings = (
                ('id', 'post_id'),
            )
        )

    def read_forums(self):
        """Read Zendesk Forums from xml file and save them as Zendesk 
        models.

        Forums in Zendesk are category groupings for forum posts. They
        do not have any "posts" themselves, but have Entries. Entries
        then have Posts:
            - Forum
                - Entry
                    - Post
                    - Post
                - Entry
                    - Post
            - Forum
                - Entry
                    - Post
            ...
            ...
        """
        self.read_xml_file(
            file_name = 'forums.xml',
            entry_name = 'forum',
            model = zendesk_models.Forum,
            fields = (
                'description', 'display-type-id',
                'entries-count', 'is-locked',
                'name', 'organization-id',
                'position', 'updated-at',
                'translation-locale-id',
                'use-for-suggestions',
                'visibility-restriction-id',
                'is-public'
            ),
            extra_field_mappings = (('id', 'forum_id'),)
        )

    def read_tickets(self):
        """Read Zendesk Tickets from xml file and save them as Zendesk 
        models.
        
        This is a little more complex in that we want to read the Comments
        as well which are child elements on the ticket. We define this with the
        sub_entities parameter.
        """
        self.read_xml_file(
            file_name = 'tickets.xml',
            entry_name = 'ticket',
            model = zendesk_models.Ticket,
            fields = (
                'assigned-at', 'assignee-id', 'base-score', 'created-at', 
                'current-collaborators','current-tags','description', 
                'due-date', 'entry-id', 'external-id', 'group-id', 
                'initially-assigned-at', 'latest-recipients', 
                'organization-id', 'original-recipient-address', 'priority-id', 
                'recipient', 'requester-id', 'resolution-time', 'solved-at', 
                'status-id', 'status-updated-at', 'subject', 'submitter-id', 
                'ticket-type-id', 'updated-at', 'updated-by-type-id', 'via-id', 
                'score', 'problem-id', 'has-incidents'
            ),
            extra_field_mappings = (('nice-id', 'ticket_id'),),
            sub_entities = [
                {'comments': (
                    zendesk_models.Comment, 
                    ['author-id', 'created-at', 'is-public', 'type', 'value', 
                        'via-id', 'ticket-id'], 
                    None
                    )
                }
            ]
        )

    @transaction.autocommit
    def import_users(self):
        """Creates new Askbot users for each zendesk_models.User. 

        For each Zendesk user, see if there are any matching Askbot users
        with the same email. If not, create a new Askbot user and copy 
        over any openauth id info as well. 

        See create_askbot_user() for a full list of fields that are copied over
        from Zendesk.
        """
        added_users = 0
        for zd_user in zendesk_models.User.objects.all():
            # if email is blank, just create a new user
            if zd_user.email == '':
                ab_user = create_askbot_user(zd_user)
                # todo: check for failure?
                if ab_user is None:
                    continue
                added_users += 1
                console.print_action(ab_user.username)
            else:
            # create new user if no matching user email was found
                try:
                    ab_user = askbot_models.User.objects.get(email = zd_user.email)
                except askbot_models.User.DoesNotExist:
                    ab_user = create_askbot_user(zd_user)
                    if ab_user is None:
                        continue
                    added_users += 1
                    console.print_action("%d %s" % (added_users, ab_user.username))
            zd_user.askbot_user_id = ab_user.id
            zd_user.save()
            # save open auth info as well.
            if zd_user.openid_url != None and \
                'askbot.deps.django_authopenid' in settings.INSTALLED_APPS:
                from askbot.deps.django_authopenid.models import UserAssociation
                from askbot.deps.django_authopenid.util import get_provider_name
                try:
                    assoc = UserAssociation(
                        user = ab_user,
                        openid_url = zd_user.openid_url,
                        provider_name = get_provider_name(zd_user.openid_url)
                    )
                    assoc.save()
                except:
                    # unsupported provider
                    pass

        console.print_action('%d users added' % added_users, nowipe = True)

    @transaction.autocommit
    def _import_posts(self, question, entry):
        """Create Askbot answers from Zendesk Entries.

        :param question: (obj) Askbot Post object with post_type='question'
        :param entry: (obj) Zendesk Entry object 
        """
        for post in zendesk_models.Post.objects.filter(
                        entry_id=entry.entry_id
                        ).order_by('created_at'):
            # create answers
            answer = post_answer(post, question=question)
            if not answer:
                continue
            post.ab_id = answer.id
            post.save()

    @transaction.autocommit
    def _import_entry(self, entry):
        """Create an Askbot question and answers from a Zendesk Entry

        :param entry: (obj) Zendesk Entry object

        :returns: (bool) True if Entry (and Posts linked to the Entry) were
        posted successfully. False if not.
        """
        question = post_question(entry)
        if not question:
            return
        entry.ab_id = question.id
        entry.save()
        self._import_posts(question, entry)
        return True

    def import_forums(self, forums, tags, date_filter):
        """Import Zendesk forums into Askbot. Create questions from Zendesk
        Entries and answers from Zendesk Posts.

        :param forums: (list) zendesk_models.Forum objects to import
        :param tags: (list) tags (str) to filter Zendesk Forum Entries by. 
        Entries that match ANY of the tags will be posted as questions. Tags 
        are case-insensitive in this import regardless of your settings in 
        Askbot.
        :param date_filter: (tuple) two-element tuple representing the start
        date and end date to filter Zendesk Forum Entries by date range. The
        tuple values are datetime objects or None.
        """
        if tags:
            print "Filtering forum posts by tags: %s" % tags
        if date_filter:
            print "Filtering forum post by dates between %s and %s" % (date_filter[0], date_filter[1])
        print "Importing forums... "
        print "="*64
        for forum in forums:
            thread_count = 0
            # don't import private forums, forums restricted to organizations
            # or forums that require login (comment this out if you don't care,
            # or modify the viewable_to_public() method for zendesk_models.Forum)
            if not forum.viewable_to_public():
                console.print_action("Skipping private forum \"%s\"" % forum.name, 
                                     nowipe = True)
                continue
            sys.stdout.write("[#%d] %s: " % (forum.forum_id, forum.name))
            for entry in zendesk_models.Entry.objects.filter(forum_id=forum.forum_id):
                # filters
                # if provided, only post entries matching ANY of the tags
                if not self._matches_tag_filter(entry.tags, tags):
                    continue
                if not self._matches_date_filter(entry.created_at, date_filter):
                    continue
                if self._import_entry(entry):
                    thread_count += 1
                console.print_action("%d threads" % thread_count)
            console.print_action("%d total threads" % thread_count, nowipe = True)

    @transaction.autocommit
    def _import_comments(self, question, ticket):
        """Import Zendesk Ticket Comments into Askbot as answers.

        :param question: (obj) askbot Post object with post_type='question' to
        create the answers for.
        :param ticket: (obj) zendesk_models.Ticket object to pull the comments
        from for creating answers.
        """
        first = True
        i=0
        for comment in zendesk_models.Comment.objects.filter(
                        ticket_id=ticket.ticket_id, is_public=True
                        ).order_by('created_at'):
            # create answers, first comment is a copy of the one on the ticket
            if first:
                first = False
                continue
            i+=1
            answer = post_answer_from_comment(comment, question=question)
            if not answer:
                continue
            comment.ab_id = answer.id
            comment.save()

    @transaction.autocommit
    def import_tickets(self, tags, date_filter):
        """Import Zendesk Tickets into Askbot as questions.

        :param tags: (list) tags (str) to filter Zendesk Tickets by. 
        Tickets that match ANY of the tags will be posted as questions. Tags 
        are case-insensitive in this import regardless of your settings in 
        Askbot.
        :param date_filter: (tuple) two-element tuple representing the start
        date and end date to filter Zendesk Tickets by date range. The
        tuple values are datetime objects or None. The date_filter is matched
        against Ticket.created_at.
        """
        # todo: optimmize with smart query
        # Ticket.objects.get(
        #     Q(created_at__gt=date_filter[0]),
        #     Q(created_at__lt=date_filter[1]),
        #     Q(tags__icontains='foo') | Q(tags__icontains='bar')
        # )
        if tags:
            print "Filtering tickets by tags: %s" % tags
        if date_filter:
            print "Filtering tickets by dates between %s and %s" % (date_filter[0], date_filter[1])
        sys.stdout.write("Importing tickets: ")
        ticket_count = 0
        for ticket in zendesk_models.Ticket.objects.all():
            # filters
            # if provided, only post entries matching ANY of the tags
            if not self._matches_tag_filter(ticket.current_tags, tags):
                continue
            if not self._matches_date_filter(ticket.created_at, date_filter):
                continue
            question = post_question_from_ticket(ticket)
            if not question:
                continue
            ticket.ab_id = question.id
            ticket.save()
            self._import_comments(question, ticket)
            ticket_count += 1
            console.print_action("%d tickets" % ticket_count)
        console.print_action("%d total tickets" % ticket_count, nowipe = True)

    def _matches_tag_filter(self, item_tags, tag_filter):
        """Determine if an item's tags satisfy the tag filter. The comparison 
        is case-insensitive.

        :param item_tags: (str) space-separated string of tags associated with
        the item.

        :param filter_tags: (str) space-separated string of tags being filtered
        for.

        :returns: (bool) True if ANY of the tags in item_tags match the tags
        in tag_filter. False if no matches are found.
        """
        if not tag_filter:
            return True
        if not item_tags:
            return False
        item_tags_list = item_tags.lower().split()
        for t in tag_filter:
            if t.lower() in item_tags_list:
                return True
        return False

    def _matches_date_filter(self, item_date, date_filter):
        """determine if an item's datetime stamp satisfies the date filter. 

        :param item_date: (datetime) generally the item's created_at datetime
        object

        :param date_filter: (tuple) pair of datetime objects representing the
        start and end dates to filter items by. If the first object is None,
        then the filter implies all items before the second datetime. Conversely,
        if the second datetime is None, the filter implies all items after the
        first datetime.

        :returns: (bool) True if date_filter is an empty tuple OR the item_date
        falls within the date_filter tuple.
        """
        if not date_filter:
            return True
        start_date = date_filter[0]
        end_date = date_filter[1]
        if not start_date:
            start_date = datetime.min
        if not end_date:
            end_date = datetime.max
        return item_date > start_date and item_date < end_date
