"""
Name: sign_task
Title:
Author: Cooper
Date: 09/09/2019
Modified: 25/08/2020

Desc:  This was originally done for hires, but now extended for standard res.  Largely mimcs sign_manager in functionality
but this is to be run on an Onion where its host is a sign, so no need for sign tables, polling the entire HCP
address range, dealing with a bunch of signs so the routines will be simplified versions.

This module will need to be able to accept data in superX format, or the format defined in template_generator:
    display_data = {
        "$bcol": "0,0,0",
        "$fcol": "0,0,0
        "$rn": "12A",
        "$dest": ["Language1_TopLine/Language1_BottomLine", "Language2"]
        }


### Cloud Updating ###

This module will support cloud updating but there are a few things that need to be mentioned.

The configurations for cloud need to be obtained via the console via a secure MQTT channel.
This will contain the necessary details to be able to connect to a secure server.

It will be up to this module to obtain said details.

"""
import os
import _thread
import serial
import time
import logging
logger = logging.getLogger("sign_task")

from hanip.debug import print_text
from hanip.onionip import hcp
from hanip.onionip import renderbox
from hanip.onionip.sign import template_generator
from hanip.onionip.sign import sign_firmloader

class SignTask(object):
    def __init__(self, config_dict, hw_dict, config_dir, data_dir):
        self.hw_dict = hw_dict
        self.config_dict = config_dict
        self.config_dir = config_dir
        self.data_dir = data_dir

        self.sign_task_loop_status = False
        self.pause = False      #Allows the main loop to be paused so that another process can communicate directly with sign
        self.stop = False       #Calling stop breaks the sign loop and thus implies the whole application needs restarting

        self.ser_enabled = False
        self.renderbox_enable = self.config_dict.get("RENDERBOX_enable", False)     #Leave renderbox disabled as a fontlib cannot be assumed if it isnt enabled via config
        self.renderbox_filter_elements = self.obtain_renderbox_command_filter()

        self.address = int(self.hw_dict["address"]) + 1
        self.resolution = self.hw_dict["sign_size"]
        self.colour_panel = self.hw_dict["colour_resolution"]
        self.new_sign_data = None           #This is how new data is input into this class
        self.sign_data = None
        self.sign_status = ["", ""]     #[StatusCode, Extended Status]
        self.sign_type = self.hw_dict["hw_type"]    #Defines whether the sign is int or ext
        self.hmf6_route_number = None

        self.blank_level = 0
        self.test_mode = False
        self.ext_test_mode = False

        self.default_page_time = 3          #Default page timer if one isnt supplied, although it doesnt look like it is used?
        self.status_poll_interval = 10       #This is how often the sign status is polled
        self.display_task_interval = 1      #This is the time before the next loop starts
        self.anti_cooper_filter = 5         #Prevents sign spamming
        self.alternative_font_path = None

        """
        Update Variables
        """
        self.wdm_enabled = False
        self.wdm_mqtt_connected = False
        self.wdm_parameter_timer = 0
        self.ctrl_loading = None

        """
        Import inits init bruv
        """
        self.hcp = hcp.HCP()
        self.rb = renderbox.RenderBox(self.config_dict)
        self.setup_graphic_library(self.config_dict.get("SIGN_int_sixteen_high", False))
        self.tg = template_generator.TemplateGenerator(self.config_dict, self.data_dir)
        self.sign_firmloader = sign_firmloader.SignFirmloader(self.hw_dict)
        self.cu = None

        try:
            if "oniondebug_st" in os.listdir("/tmp"):
                self.debug = True
            else:
                self.debug = False
        except FileNotFoundError:
            self.debug = False

    """
    ###################################################################################################################
    Setup
    """
    def setup_graphic_library(self, forced_16: False):
        """
        This used to rely on the sign graphics file but as that has not changed in many versions and to simply things
        it is now based in sign task for all modules that import this module.

        Any additions/changes here should be reflected in signApp if appropriate.

        Thanks to RayRay and his shenanigans, I now have to support 16 high mode.
        """
        nineteen_high_mode = "-"
        sixteen_high_mode = "+"

        if forced_16:
            logging.info("ST: Internal graphics in 16 high mode")
            mode = sixteen_high_mode
            padding = ""
        else:
            mode = nineteen_high_mode
            padding = "00"

        # CHECK changes signApp!!!
        self.graphic_dict = {
            "SQUARE_ext": r"00{\mode0\mss\at\al\pic\picw3\pich3 070507}",
            "SQUARE_int": r"00\$MODE\s\g0700$PAD0500$PAD0700$PADg\;".replace("$MODE", mode).replace("$PAD", padding),
            "TRI-UP_ext": r"00{\mode0\mss\at\al\pic\picw3\pich3 070301}",
            "TRI-UP_int": r"00\$MODE\s\g0700$PAD0300$PAD0100$PADg\;".replace("$MODE", mode).replace("$PAD", padding),
            "TRI-DOWN_ext": r"00{\mode0\mss\at\al\pic\picw3\pich3 040607}",
            "TRI-DOWN_int": r"00\$MODE\s\g0400$PAD0600$PAD0700$PADg\;".replace("$MODE", mode).replace("$PAD", padding),
            "ARROW-UP_ext": r"00{\mode0\mss\at\al\pic\picw3\pich3 070101}",
            "ARROW-UP_int": r"00\$MODE\s\g0700$PAD0300$PAD0100$PADg\;".replace("$MODE", mode).replace("$PAD", padding),
            "ARROW-DOWN_ext": r"00{\mode0\mss\at\al\pic\picw3\pich3 040707}",
            "ARROW-DOWN_int": r"00\$MODE\s\g0400$PAD0600$PAD0700$PADg\;".replace("$MODE", mode).replace("$PAD", padding),
            "ERROR_X_ext": r"00{\mode0\mss\at\al\pic\picw3\pich3 050205}",
            "ERROR_X_int": r"00\$MODE\s\g0500$PAD0200$PAD0500$PADg\;".replace("$MODE", mode).replace("$PAD", padding),
            "DIAMOND_ext": r"00{\mode0\mss\at\al\pic\picw3\pich3 020702}",
            "DIAMOND_int": r"00\$MODE\s\g0200$PAD0700$PAD0200$PADg\;".replace("$MODE", mode).replace("$PAD", padding),
        }

    """
    ###################################################################################################################
    Serial Port
    """
    def init_serial(self):
        """
        Initiates the serial port to communicate with the sign and then sets ser_enabled flag to True
        """
        try:
            self.ser = serial.Serial(
                self.config_dict.get("SERIAL_unix_comport", "/dev/ttyS1"),
                self.config_dict.get("SERIAL_baudrate_host_sign", 38400),
                timeout= 0.5
            )

            self.ser_enabled = True
        except serial.SerialException:
            pass

    def close_serial(self):
        """
        Closes the serial port and sets the ser_enabled flag to False
        """
        if self.ser_enabled:
            self.ser.close()
            self.ser_enabled = False

    def transmit_message(self, message, numofbytes=0):
        """
        Encodes the message into a HCP packet before sending over the serial port.  If there is a reply then it is
        obtained here due to the speed of the replies
        :param message: Message to be encoded into HCP and sent
        :param numofbytes: Number of bytes to read on the serial port, if zero then it is skipped
        :return: If there are bytes to read, return those, otherwise returns an empty string
        """
        if message == None:
            return ""

        if self.ser_enabled:
            if message[0] == "\x02":
                encodedMsg = message
            else:
                encodedMsg = self.hcp.encodeMaster(message)

            if self.debug:
                print(print_text.PrintText.to_ascii(encodedMsg))

            # logging.debug(print_text.PrintText.to_ascii(encodedMsg))

            if type(encodedMsg) is bytes:
                self.ser.write(encodedMsg)
            else:
                self.ser.write(encodedMsg.encode("latin-1"))

            if numofbytes > 0:
                reply = self.ser.read(numofbytes)
                if reply == None:
                    return ""
                else:
                    logging.debug(print_text.PrintText.to_ascii(reply))
                    return reply.decode("latin-1")
            else:
                return ""
        else:
            logging.warning("ST: Serial port not enabled")

    """
    ###################################################################################################################
    WDM Handling
    """
    def init_wdm(self, wdm_parameters):
        """
        Initialises the WDM client, because the details of the WDM server are unknown initially until the successful
        connection to a broker supplying said credentials then we cannot really start up the client, or there isn't much
        point in doing so.

        So we would need to setup after we have obtained some server information
        """
        logging.info("ST: Starting WDM")

        from hanip.hanover_cloud import wdm_client
        self.wdm = wdm_client.WDMClient(self.hw_dict, self.data_dir)
        self.wdm.update_wdm_configurations(wdm_parameters)
        self.wdm.setup_ftp_client()
        self.set_wdm_payload_attributes()
        self.wdm.allow_download = wdm_parameters.get("updates_permitted", False)

        _thread.start_new_thread(self.wdm.run, ())
        self.wdm_enabled = True

    def init_wdm_cred_mqtt_client(self):
        """
        Sets up the MQTT client to be able to recieve secure MQTT credentials over MQTT.
        """
        logging.info("ST: Setting up secure MQTT client")
        from hanip.hanover_cloud import wdm_credential_service
        self.wdm_cred_serv = wdm_credential_service.WDM_Credential_Service_Sign(self.config_dict, self.hw_dict)

        self.wdm_cred_serv.find_connect_subscribe_to_service()


    def set_wdm_payload_attributes(self):
        """
        This tells the WDM module what sort of payloads to obtain along with all the necessary bits of information
        that it needs to process the payloads
        """
        cloud_paths = {
            "status_path": r"/vehicle/<UNIT_ID>/<SIGN_ADDRESS>/log".replace("<SIGN_ADDRESS>", self.hw_dict["address"]),
            "token_root_path": r"/vehicle/<UNIT_ID>/<SIGN_ADDRESS>".replace("<SIGN_ADDRESS>",
                                                                                self.hw_dict["address"])
        }

        payload_types = {
            "onion_config": {
                "token": "ONION_CONFIG_FILE",
                "file_name": "config.cfg",
                "file_local_path": "/etc/hanip/config.cfg",
                "transfer": False
            },
            "fontlib": {
                "token": "FONTLIB_FILE",
                "file_name": self.config_dict.get("RENDERBOX_fontlib", "fontlib.bin"),
                "file_local_path": "/usr/share/renderbox/fontlib.bin",
                "transfer": False
            },
            "sign_firmware": {
                "token": "SIGN_FIRMWARE_FILE",
                "file_name": "signfirmware.bin",
                "file_local_path": None,
                "transfer": True
            },
            "onion_firmware": {
                "token": "ONION_FIRMWARE_FILE",
                "file_name": "hanip_update.whl",
                "file_local_path": None,
                "transfer": True        # This is an exception
            },
        }

        self.wdm.set_payload_attributes(payload_types)
        self.wdm.set_cloud_paths(cloud_paths)

    def check_wdm(self):
        """
        This is the entry point into the WDM module be it checking whether there is a payload, and updating the console
        if there is one.

        This doesnt need to use any of the components based in xfer because WDM provides a dictionary of the available updates
        All payloads will be in /tmp

        The console provide sign health but not sure how this would work
        """
        if self.wdm.updates_available:
            logging.info("ST: WDM updates available")
            update_flags = self.wdm.importing_module_flags
            if update_flags != None:
                self.process_wdm_updates(update_flags)

    def process_wdm_updates(self, update_flags):
        """
        Processes the flags from wdm_client and deals with the updates appropriately.  Files to handle:
        1) Sign firmware
        2) Onion configs (app needs to restart)
        """
        # print(update_flags)

        reboot_required = False

        for payload_type, payload_details in update_flags.copy().items():
            payload_path = os.path.join("/tmp", payload_details["name"])

            #Deal with config file
            if payload_type == "onion_config":
                if payload_details["ready"] and payload_details["loaded"]:
                    logging.info("ST: New config received: %s" % payload_type)
                    self.delete_network_flag()
                    reboot_required = True

            #Deal with sign firmware
            if payload_type == "sign_firmware":
                if payload_details["ready"] and payload_details["loaded"] is False:
                    logging.info("CT: Processing %s" % payload_type)

                    payload_path = os.path.join("/tmp", payload_details["name"])
                    status = self.update_firmware(payload_path)
                    if status:
                        # I guess if it fails, it will try it again on the next loop or do we reboot after an attempt?
                        self.wdm.update_importer_task_flag(payload_type, True)
                        reboot_required = True

            # This should be identical on console task along with handle_onion_firmware_payload()
            if payload_type == "onion_firmware":
                if payload_details["ready"] and payload_details["loaded"] is False:
                    if self.handle_onion_firmware_payload():
                        self.wdm.update_importer_task_flag(payload_type, True)
                elif payload_details["ready"] and payload_details["loaded"]:
                    reboot_required = True

        if reboot_required:
            self.wait_for_wdm_to_complete()
            self.stop = True


    def handle_onion_firmware_payload(self):
        """
        Between this and OAU there are a couple of flags that are taken into account so that both sides know what is
        going on.

        OAU will not apply an update without a trigger file, this trigger file also instructs OAU not to reboot the
        application either so it will sit and wait until it can continue.

        Before the application restarts, the oau_done_flag must be cleared.
        """
        oau_trigger_path = "/tmp/wdm_trigger"
        oau_done_flag = "/tmp/OAU_FIRMWARE_COMPLETE"
        update_done = False

        oau_trigger_exists = os.path.isfile(oau_trigger_path)
        oau_done_exists = os.path.isfile(oau_done_flag)

        if not oau_trigger_exists and not oau_done_exists:
            # If neither flags are present then set the trigger
            with open(oau_trigger_path, "w") as trigger:
                pass
        elif oau_trigger_exists and not oau_done_exists:
            # Trigger is there, just waiting for OAU to finish
            logging.info("ST: Waiting for OAU")

        elif oau_done_exists:
            # OA has finished
            logging.info("ST: OAU reports update complete")
            try:
                os.remove(oau_done_flag)
            except OSError:
                logging.warning("ST: Cannot remove OAU done flag")

            update_done = True

        return update_done

    def wait_for_wdm_to_complete(self):
        """
        This allows a wait period for WDM to complete doing what it needs to do before the application stops itself
        """

        if self.wdm_enabled:
            print(self.wdm.updates_available)

            logging.info("ST: Waiting for WDM to complete tasks")
            while 1:
                if self.wdm.updates_available:
                    time.sleep(1)
                else:
                    break


    def delete_network_flag(self):
        """
        Deletes the network flag so that new network settings can be obtained
        """
        try:
            os.remove(os.path.join(self.config_dir, "network.done"))
            logging.error("ST: Deleting network flag")
        except OSError:
            logging.error("Cannot delete network flag")

    """
    ###################################################################################################################
    Sign firmware/file updating
    """
    def set_cu_busy(self, state):
        """
        Sets a flag to tell config updater not to accept anything else
        """
        if self.cu is not None:
            if state:
                self.cu.update_in_process = True
            else:
                self.cu.update_in_process = False

    def update_firmware(self, firmware_file_path):
        """
        Attempts to update the sign firmware, this is somewhat trickier because the original plan was to set some
        flags and then just let it install on the next app restart but annoyingly this will reset the WDM state machine.

        So if we adopt the original plan then there needs to be a way to
        """
        # Stop and wait for the sign_task loop to be paused before proceeding
        self.pause = True
        self.set_cu_busy(True)

        while self.sign_task_loop_status:
            time.sleep(1)

        status = self.sign_firmloader.update_firmware(firmware_file_path)

        if status == "SUCCESS":
            logging.info("ST: Sign firmware install successful")
        else:
            logging.warning("ST: Sign firmware install unsuccessful, will attempt reinstall at next boot")

        # Unpause the sign test loop
        self.pause = False
        self.set_cu_busy(False)
        return status

    def update_72k_file(self, file_path):
        """
        This sends the file to the 72k and only works for 72k based processors.

        Support needed for 7113x processors also?
        """
        self.pause = True
        self.set_cu_busy(True)
        status = "FAIL"

        if "colems-72" in self.hw_dict["software_version"].lower():
            #Additional check just incase
            file_name = os.path.split(file_path)[-1]
            data_type = None

            if file_name == "fontlib.wbin":
                data_type = "font"
            elif file_name == "config.ini":
                data_type = "config-ini"

            if data_type is not None:
                return_code, status = self.sign_firmloader.call_firmloader(data_type, file_path)

                if return_code == 1:
                    logging.warning(status)
                else:
                    status = "SUCCESS"

            else:
                logging.warning("ST: File is not suitable for 72k")

        else:
            logging.warning("ST: This file is not suitable for the host")

        self.pause = False
        self.set_cu_busy(False)

        return status


    """
    ###################################################################################################################
    Status Polling
    """
    def get_status(self, extended=False):
        """
        Work in progress
        :param extended:
        :return:
        """
        #TODO: Colour support
        if extended:
            reply = self.transmit_message("9%s" % self.address, 100)
        else:
            reply = self.transmit_message("2%s" % self.address, 8)

        if len(reply) > 0:
            reply = reply[3:-3]
            return reply
        else:
            return None

    def update_status_list(self):
        """
        Updates the status of the sign_status list with both the short status and the extended
        Example:
        ['00', 'OLEMS V1.30.03 X2.2 #0 160x24 [00] C=5112 P=15/15 A2C8']
        """
        self.sign_status = ["NA", "NA"]
        self.sign_status[0] = self.get_status()
        self.sign_status[1] = self.get_status(True)

        logging.info(self.sign_status)

    """
    ###################################################################################################################
    Other sign stuff
    """
    def handle_brightness(self, level):
        """
        Sends the appropriate brightness related values to the signs i.e. Min and Max.  Gain is not currently supported
        Three brightness levels:
        0: Normal operation
        1: Signs dimmed to reduce power draw
        2: Signs blanked
        :param level: Level of brightness
        """
        try:
            min, max, gain = self.config_dict["BRIGHTNESS_sign%s" % self.hw_dict["address"]].split(",")
        except (KeyError, ValueError):
            logging.warning("ST: Cannot parse individual brightness")
            min = self.config_dict.get("BRIGHTNESS_min_brightness", 5)
            max = self.config_dict.get("BRIGHTNESS_max_brightness", 100)
            gain = self.config_dict.get("BRIGHTNESS_brightness_gain", 10)

        if level == 1:
            blanking_level = self.config_dict.get("ECOMODE_blanking_level", 30)
            msg = "90SC=MB%s;MINB%s" % (blanking_level, min)
        elif level == 2:
            msg = "C0"
        else:   #Equivalent to level 0
            msg = "90SC=MB%s;MINB%s;BG%s" % (max, min, gain)

        logging.info("ST: %s" % msg)
        self.transmit_message(msg, 0)

    def display_sign_graphic(self, choice):
        """
        Transmits the sign graphic of choice, if the requested graphic cannot be found then sends two dashes instead
        as a placeholder.  Showing something is better than nothing (I think).
        :param choice: Sign graphic to send
        """
        try:
            graphic = self.graphic_dict["%s_%s" % (choice,self.hw_dict["hw_type"])]
        except KeyError:
            print("Requested sign graphic doesnt exist", choice, self.hw_dict["hw_type"])
            graphic = "00--"

        self.transmit_message(graphic, 0)

    def clear_sign(self, clearSignData=False):
        """
        Clears the sign (HMFC)
        """
        if clearSignData:
            self.sign_data = ""
            self.new_sign_data = None

        msg = "C0"
        self.transmit_message(msg, 0)

    def set_route_number(self, route_number):
        """
        A means to set the broadcast route number for legacy graphic, but also when superX and renderbox is involved
        then it will need to be replaced.

        This value can be set to None or empty, in which case it won't be transmitted
        """

        self.hmf6_route_number = route_number
        logging.info("ST: HMF6 set to %s" % route_number)

    def broadcast_hmf6(self):
        """
        This broadcasts the current route number for programmable RN stuff
        :return:
        """

        if self.hmf6_route_number is not None and self.hmf6_route_number != "":
            # route_number = self.hmf6_route_number.zfill(4)
            route_number = self.hmf6_route_number.lstrip("0")
            hmf6 = "60" + route_number
            logging.info("ST: HMF6 %s" % route_number)
            self.transmit_message(hmf6, 0)

    """
    ###################################################################################################################
    Data handling
    """
    def update_data(self, data, render=True):
        """
        This is the entry point of the module.  This can accept any valid form of sign message be it SuperX or legacy
        graphic, or messages already encapsulated in HCP

        :param data: SuperX text or graphic, Legacy graphic
        """
        try:
            if render:
                sign_data = self.render_message(data)
            else:
                sign_data = data

            if self.new_sign_data != sign_data:
                self.new_sign_data = sign_data
                logging.info("ST: Data updated!")
            else:
                logging.info("ST: Data identical")
        
        except Exception as e:
            logging.exception("ST: Error rendering message")

    def update_data_dict(self, data):
        """
        This is the entry point of the module if it is being given a dictionary of display data (See top of module).
        It will then generate an appropriate template for the size of sign and render it and then update new_sign_data
        with the result.

        This takes into account whether it is dealing with an external or internal sign.
        :param data: display_data dictionary
        """
        if self.sign_type == "ext":
            sign_dict = {
                "sign0": {
                    "resolution": self.resolution,
                    "colour_panel": self.colour_panel,
                }
            }

            sign_data = self.tg.process_display_text(data, sign_dict)[0]
        else:
            sign_data = self.tg.process_internal_display_text(data)

        sign_data_rendered = self.render_message(sign_data)
        self.new_sign_data = sign_data_rendered

        logging.info("ST: Data updated!")

    def process_message_for_rendering(self, message):
        """
        As the message needs to go through several stages of "cleansing", it makes sense for a single
        routine to handle it all.

        If additional needs are required then they can be handled here instead of changing other bits to cater
        """

        #Firstly strip HCP stuff:
        stripped_msg = self.strip_hcp_control_chars(message)

        #Then substitute in the route number if appropriate:
        #RN when obtained from the console is 4 digits with leading zeros (usually)
        if r"\prn" in message:
            logging.info("ST: Substituting prn")
            if self.hmf6_route_number is not None:
                normalised_rn = self.hmf6_route_number.lstrip("0")
            else:
                normalised_rn = " "
            substituted_message = stripped_msg.replace(r"\prn", normalised_rn)

            return substituted_message

        else:
            return stripped_msg


    def strip_hcp_control_chars(self, hcp_message: str) -> str:
        """
        This will strip the HCP control chars so that a message can be massaged into the right shape.
        Once massaged the message will need the control chars readded and a new checksum
        """
        if hcp_message[0] == "\x02":
            logging.info("ST: Stripping hcp chars")
            stripped_msg = hcp_message[3:-3]
        else:
            stripped_msg = hcp_message

        return stripped_msg

    def process_non_latin_msg(self, message):
        """
        In the case that a UTF-8 message is received with non latin characters and renderbox is disabled or broken
        """
        try:
            message.encode("latin-1")
        except UnicodeEncodeError:
            stripped_message = self.strip_hcp_control_chars(message)

            msg = stripped_message.encode("latin-1", "replace").decode("latin-1")
            return msg
        else:
            return message

    def look_for_fontlib(self):
        """
        This deals with the potential inconsistencies with the name of the fontlib
        """
        configured_fontlib_path = os.path.join(self.data_dir, "renderbox", self.config_dict.get("RENDERBOX_fontlib", "fontlib.bin"))

        if os.path.isfile(configured_fontlib_path):
            return configured_fontlib_path
        else:
            if self.alternative_font_path is None:
                fontlib_location = os.path.join(self.data_dir, "renderbox")
                files = os.listdir(fontlib_location)

                for potential_font_file in files:
                    #Just grab the first file with fontlib name lol
                    if "fontlib" in potential_font_file.lower():
                        alternative_fontlib = os.path.join(self.data_dir, "renderbox", potential_font_file)
                        logging.info("ST: Using alternative fontlib %s" % alternative_fontlib)
                        self.alternative_font_path = alternative_fontlib
                        return alternative_fontlib

                logging.warning("ST: Cannot find alternative font...")
                return ""

            else:
                return self.alternative_font_path

    def obtain_renderbox_command_filter(self):
        """
        This obtains whatever filters needed to get stuff to display on the sign properly
        """
        raw_elements = self.config_dict.get("RENDERBOX_command_filter", None)

        if raw_elements is None:
            return None

        elements = raw_elements.split(";")

        if len(elements) > 0:
            return elements
        else:
            return None

    def render_message(self, message):
        """
        Renders the SuperX Text into equivalent SuperX Graphic only for external signs

        To cover all weird combinations of things, this can accept input from:
        - Template generator where only the SuperX data is contained
        - A complete HCP message, either SuperX text or graphic
        This will not check whether a message is valid though!!!

        There are cases where the input is not compatible with renderbox, or does not need rendering.

        In order to support programmable route numbers, this needs to replace \prn with an available route number.

        :param message: SuperX text
        :return: Render of the message if enabled, otherwise the input message is returned untouched.
        """
        #Check whether the message needs rendering at all
        if self.hw_dict["hw_type"] == "int":
            return_message = self.render_internal_message(message)
        elif "\picw" in message:
            logging.info("ST: Graphic received, skipping render")
            return_message = message
        elif message[0] == "\x02" and message[1] == "1":
            logging.info("ST: Legacy graphic, skipping render")
            return_message = message

        elif self.renderbox_enable:
            logging.info("ST: Rendering graphic %s" % ("sxtrans" if self.config_dict.get("RENDERBOX_use_sxtrans", False) else "xt"))
            logging.info("\tfilter: %s" % self.renderbox_filter_elements)
            render = self.rb.getSuperXRender(
                self.hw_dict["sign_size"],
                self.look_for_fontlib(),
                self.config_dict.get("RENDERBOX_font_mapping", "e66:50"),
                self.process_message_for_rendering(message),
                self.config_dict.get("RENDERBOX_use_sxtrans", False),
                self.renderbox_filter_elements            # Filter
            )

            if len(render) < 1:
                #If the fontlib is missing, then the renderer returns an empty string (see renderbox.py)
                logging.warning("ST: Empty render...")
                return_message = message
            elif "Error" in render:
                logging.error("ST: Render error :( \n\t%s" % render)
                return_message = self.process_non_latin_msg(message)
            else:
                return_message = render
        else:
            #If the message meets the requirements to be rendered but renderbox is disabled
            return_message = self.process_non_latin_msg(message)

        #Lastly append the HCP control characters if they are missing.  We can broadcast.
        if return_message[0] != "\x02" and return_message[0] != "0":
            return_message = "00" + return_message

        return return_message

    def render_internal_message(self, message):
        """
        TODO: This bit
        """
        return message

    def get_test_message(self):
        """
        Obtains the test message which is either the standard (HMF3) or the extended one which shows information related
        to the Onion.  The extended one is more appropriate for IP connected signs where we would want information such
        as software version, serial number and IP address.
        :return: standard test or extended test message
        """
        sign_size = self.hw_dict["sign_size"]
        graphic = None

        if self.ext_test_mode:
            if self.hw_dict["hw_type"] == "int":
                msg = r"00\=\s\0SW: %s\1\sSN: %s\P\p\p\d" \
                       r"\=\s\0IP: %s\1\s%s #%s\P\p\p\d" \
                       r"\=\s\0%s\1\seco: %s\P\p\p\d" % (
                    self.hw_dict["onion_ver"], self.hw_dict["serial_number"], self.hw_dict["unit_IP"],
                    self.hw_dict["sign_size"], self.hw_dict["address"],
                    self.config_dict.get("CONFIG_name", "NA"), self.blank_level
                )
            else:
                test_pattern_template = r"{\mode0\pt30\mss\at\al%s\pic\picw%s\pich%s %s}"
                try:
                    width, height = sign_size.split("x")
                except AttributeError:
                    pass
                else:
                    bytes_per_column = int(height) // 8 + (int(height) % 8 > 0)
                    graphic = "55" * bytes_per_column * int(width)

                msg = r"00{\mode2\pt30\fit2\fb0{SW: %s}\fs\fit2\fb0{SN: %s}}" \
                     r"{\mode2\pt30\fit2\fb0{IP: %s}\fs\fit2\fb0{%s #%s}}" \
                     r"{\mode2\pt30\fit2\fb0{%s}\fs\fit2\fb0{eco: %s}}" % (
                         self.hw_dict["onion_ver"], self.hw_dict["serial_number"], self.hw_dict["unit_IP"],
                         self.hw_dict["sign_size"], self.hw_dict["address"],
                         self.config_dict.get("CONFIG_name", "NA"), self.blank_level
                         )

                if graphic is not None:
                    msg += test_pattern_template % ("", width, height, graphic)
                    msg += test_pattern_template % ("\inv", width, height, graphic)

        else:
            msg = "30"

        return msg

    def extended_status_command(self, command):
        """
        *NOT JET IMPLEMENTED*
        When we want to issue HMF9 commands
        :param command:
        :return:
        """
        pass

    def get_udp_config_info(self):
        """
        For UDP based signs where it is difficult to ascertain what it is currently configured with, this gets the
        relevant data to be shown onto the sign prior to operation
        """
        if self.hw_dict["hw_type"] == "int":
            diag_message = r"0%s\=\s\0SW: %s\1\s%s%s:%s\P\p\p\d" % (self.address, self.hw_dict["onion_ver"],
                                                                 "D" if self.config_dict["NETWORK_dhcp_client"] else "S",
                                                                 self.hw_dict["unit_IP"],
                                                                 self.config_dict.get("UDP_port", 1492)
                                                                   )
        else:
            diag_message = r"0%s{\mode2{SW: %s}\fs{%s%s:%s}}" % (self.address, self.hw_dict["onion_ver"],
                                                               "D" if self.config_dict["NETWORK_dhcp_client"] else "S",
                                                                self.hw_dict["unit_IP"],
                                                                self.config_dict.get("UDP_port", 1492)
                                                                )

        return diag_message

    """
    ###################################################################################################################
    Config updater related calls
    """
    def setup_config_updater(self):
        """
        Sets up the config_updater for certain modes
        """
        from hanip.onionip import config_updater
        self.cu = config_updater.ConfigUpdater(self.config_dict, self.hw_dict, self.config_dir)
        self.cu.setup_webserver()

    def check_sign_test_trigger(self):
        """
        Checks for the presence of the sign test trigger placed by config_updater, this will remain in this routine
        until the 90s expires
        """
        if "trigger_sign_test" in os.listdir("/tmp"):
            self.ext_test_mode = True

            for seconds in range(90):
                if seconds%10 == 0:
                    msg = self.get_test_message()
                    self.transmit_message(msg, 0)
                time.sleep(1)

            self.ext_test_mode = False
            self.delete_sign_test_trigger()
            self.clear_sign()

    def delete_sign_test_trigger(self):
        """
        Deletes the sign test trigger from /tmp
        """
        logging.info("ST: Stopping triggered sign test")
        try:
            os.remove("/tmp/trigger_sign_test")
        except OSError:
            logging.exception("ST: Cannot delete sign test trigger")

    def check_sign_firmware_update(self):
        """
        Only to be used if config_updater is imported in this module.
        """
        try:
            if self.cu.new_sign_firmware:
                status = self.update_firmware(self.cu.signfirmware_path)
                return  status
        except AttributeError:
            logging.exception("ST: CU not initiated!")
            return "NOTINIT"


    """
    ###################################################################################################################
    Main
    """
    def run_wdm_loop(self):
        """
        To separate WDM stuff from sign stuff as the main loop is rather busy as it stands.
        Check that a connection has been established to a broker first:
        Once connection has been established setup WDM client, and whatever other inits that are needed.

        Then just sit and wait for new payloads
        """
        loop_wait_time = 5
        wdm_setup = False

        # Delay to allow the setup to complete as it's run in a thread as well...
        time.sleep(5)
        logging.info("ST: WDM loop running!")

        if self.config_dict.get("WDM_use_config_file", False):
            logging.info("ST: WDM using config.cfg values")
            settings = {
                "unit_id": self.config_dict.get("WDM_unit_id", ""),
                "username": self.config_dict.get("WDM_ftp_server_user_name", ""),
                "password": self.config_dict.get("WDM_ftp_server_password", ""),
                "server_ip": self.config_dict.get("WDM_ftp_server_ip", "").lower(),
                "server_port": self.config_dict.get("WDM_ftp_port", 2022),
                "holdoff": self.config_dict.get("WDM_ftp_server_delay_wait"),
                "enable_tls": self.config_dict.get("WDM_enable_tls", False),
                "updates_permitted": True,
                "scheme": "2"
            }
        else:
            logging.info("ST: WDM using MQTT values")

            while 1:
                # This checks whether there is a payload from the credentials service
                if self.wdm_cred_serv.broker_connected:
                    settings = self.wdm_cred_serv.check_new_payload()

                    if settings is None:
                        time.sleep(1)
                    else:
                        break

        print(settings)

        important_values = {settings["unit_id"], settings["username"], settings["password"], settings["server_ip"]}
        if "" in important_values:
            #Basically if any of the above are empty we cannot proceed.
            logging.warning("WDM: Missing config parameters")
        else:
            self.init_wdm(settings)
            wdm_setup = True

        #Main loop
        while 1:
            if self.wdm_enabled and wdm_setup:
                if not self.config_dict.get("WDM_use_config_file", False):
                    settings = self.wdm_cred_serv.check_new_payload()
                    if settings is not None:
                        self.wdm.update_wdm_configurations(settings)
                        wdm_enable_download = settings.get("updates_permitted", True)
                        self.wdm.allow_download = wdm_enable_download

                self.check_wdm()

            for seconds in range(loop_wait_time):
                time.sleep(1)

    def run(self):
        """
        This is the main sign control loop which can be called as a thread.

        It isnt completely necessary to use this if it isnt required, as the sign can also be poked directly by calling
        the required commands in this module.
        """
        if not self.ser_enabled:
            #At times the serial might already be enabled by the importing class
            self.init_serial()

        status_timer = time.time()
        display_timer = time.time()
        ext_test_timer = display_timer

        if not self.config_dict.get("WDM_use_config_file", False):
            _thread.start_new_thread(self.init_wdm_cred_mqtt_client, ())
        _thread.start_new_thread(self.run_wdm_loop, ())

        logging.info("ST: Running!!")
        while 1:
            logging.info("ST: Loop cycle")
            while self.pause:
                self.sign_task_loop_status = False
                logging.info("ST: Paused")
                time.sleep(self.display_task_interval)
                continue

            self.sign_task_loop_status = True

            if (time.time() - status_timer) > self.status_poll_interval:
                self.update_status_list()
                status_timer = time.time()

            #Sign tests always has priority regardless of any other incomming message
            self.check_sign_test_trigger()
            if self.test_mode or self.ext_test_mode:
                logging.info("ST: Sign Test")
                if (time.time() - ext_test_timer) > self.anti_cooper_filter*2:
                    ext_test_timer = time.time()
                    msg = self.get_test_message()
                    self.transmit_message(msg, 0)

                time.sleep(self.display_task_interval)
                display_timer = 0
                continue

            #Handle EcoMode stuff
            if self.blank_level == 2:
                self.handle_brightness(self.blank_level)
                time.sleep(self.display_task_interval)
                logging.info("ST: Signs blanked")
                continue

            if self.sign_data == None and self.new_sign_data == None:
                logging.info("ST: No sign data")
                time.sleep(self.display_task_interval)
                continue

            if self.sign_data != self.new_sign_data:
                self.sign_data = self.new_sign_data
                logging.info("ST: New sign data")
                display_timer = time.time()
                self.transmit_message(self.sign_data, 0)
            elif (time.time() - display_timer) > self.anti_cooper_filter:
                display_timer = time.time()
                self.handle_brightness(self.blank_level)        #Handle brightness stuff
                self.broadcast_hmf6()
                #ONION-75 - Allow the option to disable retransmit.  Defaults to True
                if self.config_dict.get("SIGN_enable_retransmit", True):
                    self.transmit_message(self.sign_data, 0)
            else:
                pass

            time.sleep(self.display_task_interval)

            if self.stop:
                break

    def run_udp_mode(self):
        """
        This is the mode to use when signs are need to listen over the UDP protocol which the G5 originally used.
        """
        self.setup_config_updater()
        #Init sign_task loop
        _thread.start_new_thread(self.run, ())

        #Init UDP Client
        from hanip.onionip import udp_client
        self.udpc = udp_client.UDPClient(self.config_dict, self.hw_dict)

        self.transmit_message(self.get_udp_config_info())
        time.sleep(5)

        self.display_sign_graphic("TRI-DOWN")

        while 1:
            self.check_sign_test_trigger()
            status = self.check_sign_firmware_update()
            if status == "SUCCESS":
                break

            message = self.udpc.poll(False)

            if message is not None:
                target_address = message[2]
                hmf = message[1]

                if target_address == "0":
                    if hmf == "3":
                        self.test_mode = True
                elif target_address == str(self.address):
                    logging.info("ST: New UDP Message")
                    # logging.info("ST: HMF%s %s" % (hmf, message[0:20]))
                    if hmf in ["0","1","4"]:
                        if len(message) == 7 and message[3] == "I":
                            #EG4 keeps sending this [STX]01I[ETX]53?  This is a message intended for HTC
                            #and it keeps overriding the display message.
                            pass
                        else:
                            self.test_mode = False
                            self.update_data(message, render=self.config_dict.get("RENDERBOX_enable", True))

                        continue

                    if hmf == "2":
                        ext_status_msg = "2%X00" % (self.address)
                        self.udpc.send_udp_response(ext_status_msg, self.udpc.server_address)

                    elif hmf == "6":
                            self.transmit_message(message)

                    elif hmf == "9":
                        ext_status_msg = "9%X%s" % (self.address, self.sign_status[1])
                        self.udpc.send_udp_response(ext_status_msg, self.udpc.server_address)

                    elif hmf == "C" or hmf == "c":
                        self.clear_sign(True)

                    else:
                        self.transmit_message(message)

                else:
                    pass

            time.sleep(0.1)


    def run_prod_mode(self):
        self.setup_config_updater()

        if self.hw_dict.get("unit_IP", None) != None:
            from hanip.itxpt import module_inventory_service
            self.mis = module_inventory_service.ModuleInventoryService(self.config_dict, self.hw_dict)
            self.mis.run()

        from hanip.itxpt import DNS_SD
        self.service_discover = DNS_SD.DNSSD_Discover("_itxpt_http._tcp.local.", "Han_prod")
        self.service_discover.run()

        self.init_serial()

        while 1:
            self.check_sign_test_trigger()

            if self.service_discover.serviceIP == "":
                self.transmit_message("00Onion ver: %s" % self.hw_dict["onion_ver"], 0)
                time.sleep(5)
                self.transmit_message("00" + self.hw_dict["unit_IP"], 0)
                time.sleep(5)
            else:
                self.transmit_message("30", 0)
                time.sleep(2)

            if self.stop:
                break
            elif self.cu.new_conf:
                break


if __name__ == "__main__":
    config_dict = {
        # "SIGN_int_sixteen_high": True,
        "RENDERBOX_enable": True,
        "RENDERBOX_fontlib": "fontlib-bino.bin",
        "RENDERBOX_font_mapping": "e66:50"
    }

    hw_dict = {
        "sign_size": "160x24",
        "colour_resolution": "40x24",
        "address": "1",
        "hw_type": "ext"
    }

    st = SignTask(config_dict, hw_dict, "/etc/hanip", "/usr/share")

    print(st.graphic_dict)

    messages = [
        "\x0200{\\mode3\\rw32{12}\\fs{Lillehammer-}\\fs\\fit2{Hamar-Vikingeskipet}}\x0322",
        r"{\mode3\rw32{12}\fs{Lillehammer-}\fs\fit2{Hamar-Vikingeskipet}}",
        r"{\mode3\rw32{12}\fs{Lillehammer-}\fs\fit2{Hamar-Vikingeskipet}",
        r"{\mode0\fit2{{\isp0\pic\picw17\pich11 0E00F2010006000400060003D0007800380038002000200020002000200030001000}}}",
    ]

    for message in messages:
        print(st.render_message(message))
