"""
Name: wdm_client.py
Title: WDM Client
Author: Cooper Chan
Date: 14/10/2021
Modified: 06/06/2024

Desc: This was the WDM client for the Onion, but now for Cloud applications.  This supports all appropriate payloads
now.

####  LEGACY WDM MODE  ####
1) Obtain the DB_FILE token from the server
    This token contains a path as to where the payload is.  The payload name is a hash.
2) Compare the value of the local token to the one obtained from the server
3) If the value is the same, do nothing, otherwise use the new value to obtain the new database
4) Once the new database is obtained, place a token within:
    FTP:\\dbhelen\vehicle\0001\log\DB_FILE\<TOKEN_CONTENTS>.GOT
5) Install the database via HANO-1
6) If sucessfully installed place a token within:
    FTP:\\dbhelen\vehicle\0001\log\DB_FILE\<TOKEN_CONTENTS>.DONE

#### New WDM/Cloud MODE ####

According to Faris this is the new behaviour of WDM

1) Check if there is a .UID file in FTP:\\dbhelen\vehicle\<VEHICLE_ID>>\log\DB_FILE
2) If it is missing:
    Put INIT Flag
    Download DB_FILE which points to the payload location and name.
3) Put GET Flag
    Download file thats defined in the DB_FILE
4) Put GOT Flag
5) Put LOAD Flag and begin the process of putting the database on the console
6) After successful load, place DONE flag

Flag description:
+----------------+----------+--------------------------------------------------------------+
| File Extension | Contents |                           Meaning                            |
+----------------+----------+--------------------------------------------------------------+
| .INIT          | RESULT=0 | Payload is new to device. Update required and begins.        |
| .INIT          | RESULT=1 | Payload matches previous update. No further update required. |
| .GET           | RESULT=0 | Beginning to download payload files.                         |
| .GET           | RESULT=1 | Payload files too big to fit on device. Stop update.         |
| .GOT           | N/A      | Download complete.                                           |
| .LOAD          | N/A      | Beginning local update using payload files.                  |
| .DONE          | N/A      | Update completed successfully                                |
+----------------+----------+--------------------------------------------------------------+

WDM also supports devices status' via another text file but implementation of this is not presently required it goes in
the following location:

The contents of the file are as follows:
    VARIANT_NAME            = DG3
    FIRMWARE_VERSION        = DERIC-G3 V1.43.08
    SUPPORTED_PAYLOAD_TYPES = CONSOLE_FIRMWARE_FILE,CONSOLE_CONFIG_FILE,DB_FILE
    UNIT_ID                 = 64104
    Addresses from DHCP  :
       IP address           = 192.168.0.20
       Subnet mask          = 255.255.255.0
       Gateway address      = 192.168.0.10
    Sign status :
       SIGN_STATUS=ADDR0=""=STATUS_NO_RESP_ERR
       SIGN_STATUS=ADDR1=""=STATUS_OK
       SIGN_STATUS=ADDR2=""=STATUS_NO_RESP_ERR
       SIGN_STATUS=ADDR3=""=STATUS_NO_RESP_ERR
       SIGN_STATUS=ADDR4=""=STATUS_OK
    Configuration Version   = 11
    Unique ID               = 430960173932575205D3FF37
    // Configurable values (and defaults) :
       CLIENT SERVER = 1
       ENABLE_DHCP = 1
       ...

### WDM Configurations ###
The recommendation is that WDM configurations are not stored in the Onion's configuration file (although if this is what
you want to do then I will not stop you).  In the Onion versions of the console software, console_task is able to ask
the console what the WDM paramteres are.  The advantage of this is:
- Settings can be changed on the console directly
- No need for several onion config.cfg files, only the console config.ini

Note, any changes to the FTP wait time, will not take into effect until the previous timer is reset,

### Enhancements ###
The previous version only supported eric.bins, this version can support as many payloads as one wants and the only thing
that needs updating is the payload attributes and the list of payloads to obtain.

This will need to run on both signs and consoles, but for signs there would be no need to transfer anything to the sign.
There will need to be some mechanism to tell console_task which files need sending to the console, need to determine
what order to do things...

All FTP functions have been stripped out and moved to another module, as far as this module is concerned it only needs
to grab and upload things, the file_transfer_protocols module will handle the rest

"""
import os
import io
import json
import shutil
import logging
import ftplib
import time

try:
    from hanip.onionip import file_transfer_protocols
except ImportError:
    import file_transfer_protocols

class FakeFile:
    def read(self, size=0):
        return ''

class WDMClient(object):
    def __init__(self, hw_dict, data_dir, debug=False):
        self.debug = debug
        self.config_dict = None
        self.hw_dict = hw_dict
        self.data_dir = data_dir

        if self.hw_dict["hw_type"] == "con":
            self.uid = self.hw_dict["model"] + "-" + self.hw_dict["serial_number"]
        else:
            self.uid = "SIGN" + "-" + self.hw_dict["serial_number"]

        #Tokens and paths
        self.local_tempoary_path = r"/tmp"                      #Place to stick files that don't need to persist
        self.token_local_path = self.local_tempoary_path        #Where tokens are downloaded to
        self.payload_attributes = None
        self.cloud_paths = None                                 #Cloud path dict

        #Update flags
        self.wait_for_importer_reset = False        #This allows the importer to reset the state machine instead of doing it automatically
        self.importer_reset_requested = False
        self.forced_update_required = False
        self.forced_update_ready = None
        self.database_too_phat = False          # This is for forced installations need a way to get a flag
        self.list_of_forced_updates = None
        self.updates_available = False
        self.importing_module_flags = None
        self.wdm_status = 2                         #This is the status of the state machine in the run section

        #Console Flags
        self.allow_download = True       #This is part of the ctrl_loading

        #WDM Settings
        self.unit_id = None
        self.timetowait = 120

        #Sign Status for Health Reporting and other such requirements, Thanks Faris.
        self.sign_statuses = None
        self.sign_firmwares = None

        #Setup the module
        self.ftp = None
        self.break_loop = False

    """
    ###################################################################################################################
    FTP/SFTP Abstraction Layer
    """
    def upload_file(self, destpath, src) -> None:
        """
        This function is needed because standard FTP deals with bytes and thus we can just upload a BytesIO object directly
        but as SFTP deals with file transfers we would need to create the file before we can upload it.
        """
        if self.ftp == None:
            return None

        if self.ftp.sftp_en:
            filename = os.path.split(destpath)[-1]
            src_file_path = "/tmp/%s" % filename
            with open(src_file_path, "wb") as tmp_file:
                try:
                    tmp_file.write(src.getbuffer())
                except AttributeError:
                    #empty buffers have no getbuffer call
                    pass

            status = self.ftp.upload_file(destpath, src_file_path)
        else:
            status = self.ftp.upload_file(destpath, src)

            if status == 2:
                filename = os.path.split(destpath)[-1]
                src_file_path = "/tmp/%s" % filename
                # with open(src_file_path, "rb", encoding="ascii") as tmp_file:
                status = self.ftp.upload_file(destpath, src_file_path)


        return status

    """
    ###################################################################################################################
    WDM PARAMETERS
    """
    def setup_ftp_client(self) -> None:
        """
        This initialises which FTP protocol to use.  If the server_ip is prefixed sftp then the protocol to use is SFTP
        this is the way it should be according to chocolate boss who knows nothing according to white boss.

        We will need to strip out that information too because it won't work with the SCP client either.

        This can only be called when there is a known server to connect to, so update the wdm_configuration first!
        It isnt the usual case that one would repeatedly swap from FTP to SFTP so reboot the app if thats the case.
        """
        ftp_host = self.config_dict["server_ip"]

        if ftp_host[0:4] == "sftp":
            logging.info("WDM: Using SFTP")
            self.ftp = file_transfer_protocols.SFTP_Client()
            self.ftp.sftp_en = True
        else:
            logging.info("WDM: Using FTP")
            self.ftp = file_transfer_protocols.FTP_Client()
            if self.config_dict.get("enable_ftps", False):
                self.ftp.ftps_en = True

        self.ftp.update_ftp_configuration(self.config_dict)

    def sudoku_module(self):
        """
        Allows the importer to break out the main loop
        """
        logging.info("WDM: Loop exit requested")
        self.break_loop = True

    def set_payload_attributes(self, payload_attributes: dict) -> None:
        """
        Allows the importing module to set the payload attributes.  This expects a dictionary in the following format:
        {
            "onion_config": {
                "token": "ONION_CONFIG_FILE",
                "file_name": "config.cfg",
                "file_local_path": "/etc/hanip/config.cfg",
                "transfer": False,
                "check_size": False
            },
            ...
        }

        Where:
                "token": name of the token on cloud
                "file_name": name of the file it should be saved as locally
                "file_local_path": where the file should be moved to
                "log_path": path of the logs on the remote server
                "transfer": this tells this module not to move the file, but to let some other module deal with it
                "check_size": the G3 consoles and Faris are size queens, should be omitted or set to false for everyone else.

        Note that in the logpath section, there must be <UNIT_ID>, this gets replaced with the actual ID.
        """
        self.payload_attributes = payload_attributes
        logging.info("WDM: Payload attributes set")

    def get_payload_attributes(self) -> dict:
        """
        Returns what's currently set as the payload attributes
        """
        return self.payload_attributes

    def obtain_payload_attributes(self, payload=None):
        """
        This defines all the token attributes such as their names, where they need to go locally, whether they need to
        be sent to the console etc.

        If this is called with no parameters then it will return a list of all the described payload types in this
        subroutine.  When more tokens are added/removed/modified then they should be changed here.  It maybe possible
        to export this as a separate config file

        The order of this dictionary matters as files are dealt with in the order they are listed
        """
        if self.payload_attributes is None:
            return None

        if payload is None:
            payload_type_list = []

            for key, values in self.payload_attributes.items():
                payload_type_list.append(key)

            return payload_type_list
        else:
            try:
                attributes = self.payload_attributes[payload]
            except KeyError:
                return None

            return attributes

    def get_supported_payloads(self):
        """
        This is needed in the status file, so its obtained here
        """
        if self.payload_attributes is None:
            return [""]

        payload_types = self.get_payload_attributes()
        supported_payloads = []

        for payload_type in payload_types:
            try:
                supported_payloads.append(self.payload_attributes[payload_type]["token"])
            except KeyError:
                pass

        return supported_payloads

    def set_cloud_paths(self, cloud_log_paths: dict) -> None:
        """
        This automatically generates all the log paths needed for the tokens, UIDs and status files

        "cloud_paths": {
            "status_path": r"/vehicle/<UNIT_ID>/<SIGN_ADDRESS>/log/".replace("<SIGN_ADDRESS>", self.hw_dict["address"]),
            "token_log_path": r"/vehicle/<UNIT_ID>/<SIGN_ADDRESS>/log/".replace("<SIGN_ADDRESS>", self.hw_dict["address"])
        }

        """
        if self.payload_attributes is None:
            return

        self.cloud_paths = cloud_log_paths
        payload_attributes = self.get_payload_attributes()

        for token in payload_attributes:
            token_root_path = cloud_log_paths["token_root_path"]
            token_name = payload_attributes[token]["token"]
            _token_log_path = token_root_path + "/log/" + token_name

            token_log_path = _token_log_path.replace("<UNIT_ID>", self.unit_id)

            payload_attributes[token]["log_path"] = token_log_path

        #Replace the payloads attributes dict here:
        self.set_payload_attributes(payload_attributes)

    def update_wdm_configurations(self, new_configurations) -> None:
        """
        Allows the WDM parameters to be updated after the class has been initialised
        This is useful for when settings are changed via the console.
        :param new_configurations: Dictionary of WDM settings
        :return: None
        """
        self.config_dict = new_configurations

        self.unit_id = new_configurations["unit_id"]
        try:
            self.timetowait = int(new_configurations["holdoff"])
        except ValueError:
            self.timetowait = 120

        #Need to update the FTP module too...
        if self.ftp != None:
            host = new_configurations["server_ip"]
            if host[0:7] == "sftp://":
                new_configurations["server_ip"] = host[7:]
            self.ftp.update_ftp_configuration(new_configurations)

        logging.info("WDM: Parameters updated")
        # print(self.config_dict)

    def generate_wdm_status(self) -> dict:
        """
        Generates a dictionary containing all the WDM parameters which can then be subsequently used to report the current
        settings of WDM and the current status of the payload
        :return: Dictionary of current wDM status
        """
        if self.ftp == None:
            return {}
        else:
            wdm_status = {
                "host": self.ftp.host,
                "port": self.ftp.port,
                "user": self.ftp.user,
                "password:": "",
                "unit_id": self.unit_id,
                "wait_interval": self.timetowait,
            }

            return wdm_status

    """
    ###################################################################################################################
    UID HANDLING
    """
    def check_payload_uid_files(self, list_of_payloads):
        """
        It would seem my understanding of UID files was wrong, there isn't one global UID file to signify any update
        but each payload will have a UID file.  Thanks Sandy.

        This checks each payload log folder for the UID file and if there is a UID file missing then
        """
        tokens_to_process = []

        for payload in list_of_payloads:
            logging.info("WDM: Checking UID %s" % payload)
            payload_attributes = self.obtain_payload_attributes(payload)
            log_path = payload_attributes["log_path"]

            if self.check_for_uid_file(log_path):
                tokens_to_process.append(payload)

        logging.debug("WDM: Tokens to obtain: %s" % (', '.join(tokens_to_process)))
        return tokens_to_process

    def update_uids(self, payloads_to_process):
        """
        This will set all the necessary UIDs at the end.  It is its own routine because if it was placed in
        self.check_all_payloads_complete then it would always it per cycle waiting for other transfers to complete.

        Whilst it is not an issue in the grand scheme of things perhaps it will use up more data than necessary.

        This only returns successful when all appropriate UIDs have been written.
        """
        success = True

        for payload, payload_status in payloads_to_process.items():
            payload_attributes = self.obtain_payload_attributes(payload)
            if payload_status["ready"] and payload_status["loaded"]:
                log_path = payload_attributes["log_path"]
                if self.write_uid_file(log_path):
                    success = False

        return success

    def check_for_uid_file(self, log_path):
        """
        Checks for the presence of a UID file, a missing one signifies that an update is available to be processed.

        Annoyingly FTP and SFTP can provide different results depending, due to the different way the UID is checked:

        In FTP mode, it will either return None if the file doesnt exist, and the actual size of the file, errors do not seem to be caught
        In SFTP mode, it will return None if the file doesnt exist, 0 if it does and 1 if there is an error

        If there is an error in SFTP mode it will skip that payload for that given cycle.

        :return: True is no UID file is available, else False
        """
        uid_path = os.path.join(log_path, self.uid)

        logging.info("WDM: UID path: %s" % uid_path)
        size = self.ftp.ftp_get_file_size(uid_path)
        uid_found = True

        if self.ftp.sftp_en:
            if size == 2:
                uid_found = False
        else:
            if size == None:
                uid_found = False

        if uid_found:
            #We want nothing to happen when a UID is found, or cannot be obtained
            logging.info("WDM: UID found/Error obtaining")
            return False
        else:
            logging.info("WDM: No UID file")
            return True


    def write_uid_file(self, log_path) -> int:
        """
        Writes a UID file to the server, this is the very last step in the update process and should only be written
        once everything else is complete
        :return: 1 if failure, 0 if success
        """
        logging.info("WDM: Uploading UID")

        uid_path = os.path.join(log_path, self.uid)
        upload_status = self.upload_file(uid_path, FakeFile())

        if upload_status == 0:
            return 0
        else:
            return 1

    """
    ###################################################################################################################
    TOKEN HANDLING
    """
    def process_payload_tokens(self, tokens_to_process) -> dict:
        """
        This will process each token one by one, first grabbing the token, the parsing the contents and grabbing the file.
        This will also set the appropriate flags for this stage of the update.

        A dictionary is created which is in similar format to the attribute dictionary, but basically contains information
        about whether a payload is downloaded to allow another subroutine to deal with it appropriately.  This dictionary has
        the following format:

        {
            "DB_FILE": {
                "ready": False,     The flle is ready to be processed
                "name": "",         The name of the downloaded file
                "md5": "",          MD5 of the downloaded file, this is inferred from the payload filename
                "loaded": False     Whether the file has been loaded or not
                "flag": "",         Flags for tokens so they are not resent
                "check_pass": True         Status of size check, only really relevant for DB_FILES
            }
        }

        """
        payloads_to_process = {}

        for payload in tokens_to_process:
            payload_status = {
                "ready": False,
                "name": "",
                "md5": "",
                "loaded": False,
                "flag": "",
                "check_pass": "",
            }

            logging.info("WDM: processing %s" % payload)
            payload_attributes = self.obtain_payload_attributes(payload)
            cloud_attributes = self.obtain_payload_attributes("cloud_paths")
            token_name = payload_attributes["token"]
            log_path = payload_attributes["log_path"]

            payloads_to_process[payload] = payload_status

            if self.get_token(token_name):
                payload_path, payload_md5, payload_extension = self.parse_token(token_name)
                if payload_path and payload_md5:
                    payload_name = payload_attributes["file_name"]

                    self.update_status_flag("INIT", log_path, payload_md5)
                    self.update_status_flag("GET", log_path, payload_md5)

                    if not self.ftp.download_file(payload_path, os.path.join(self.local_tempoary_path, payload_name), 1):
                        #If a download fails it will just try again on the next cycle
                        payloads_to_process[payload]["ready"] = True
                        payloads_to_process[payload]["name"] = payload_name
                        payloads_to_process[payload]["md5"] = payload_md5
                        payloads_to_process[payload]["loaded"] = False
                        if not payload_attributes.get("check_size", False):
                            self.update_status_flag("GOT", log_path, payload_md5)
                            #We need to set this flag so that we do not constantly upload LOAD flags in the next bit, this is the only flag that matters
                            payloads_to_process[payload]["flag"] = "GOT"
                        else:
                            payloads_to_process[payload]["flag"] = "GET"
                            # payloads_to_process[payload]["check_pass"] = False

        logging.debug(json.dumps(payloads_to_process))
        return payloads_to_process

    def get_token(self, token_name) -> bool:
        """
        Downloads the token from the server
        :return:
        """
        logging.info("WDM: Obtaining Token %s" % token_name)

        try:
            token_remote_path = self.cloud_paths["token_root_path"].replace("<UNIT_ID>", self.unit_id)
        except KeyError:
            #Use scheme 1 if not defined in config
            token_remote_path = r"/vehicle/%s" % self.unit_id

        remote_path = os.path.join(token_remote_path, token_name)
        local_path = os.path.join(self.token_local_path, token_name)
        if not self.ftp.download_file(remote_path, local_path):
            return True
        else:
            return False

    def parse_token(self, token_name):
        """
        Parses the contents of the token to obtain the payload location
        :return:
        """
        local_path = os.path.join(self.token_local_path, token_name)
        logging.info("WDM: Parsing Token %s" % local_path)

        with open(local_path, "r") as token:
            remote_file_location = token.read()

        try:
            file_md5 = remote_file_location.split("/")[-1][:-4]
            file_extension = remote_file_location.split("/")[-1][-4:].lower()
        except Exception as e:
            logging.warning(f"invalid path from token: {e}")
            remote_file_location, file_md5, file_extension = "", "", ""

        logging.info("\t" + remote_file_location)
        return remote_file_location, file_md5, file_extension


    """
    ###################################################################################################################
    STATUS FLAGS
    """

    def update_status_flag(self, flag_suffix, remote_token_path, payload_md5, result = None) -> int:
        """
        Uploads a flag to the server to indicate the status of the update process.  Details of the flags are listed at
        the top of this script
        Flag suffixes are one of:
        - INIT
        - GET
        - GOT
        - LOAD
        - DONE
        :param flag_suffix:
        :return: 0 if failure, 1 if success
        """
        flag_suffixes = ["INIT", "GET", "GOT", "LOAD", "DONE"]

        if flag_suffix in flag_suffixes:
            flag_path = remote_token_path + "/" + payload_md5 + "." + flag_suffix
            logging.info("WDM: Setting %s    result %s" % (flag_path, result))

            if result is None:
                if not self.upload_file(flag_path, FakeFile()):
                    return 1
            else:
                flag_file = self.generate_status_flag(flag_suffix, payload_md5, result)
                if not self.upload_file(flag_path, flag_file):
                    return 1

        return 0

    def generate_status_flag(self, flag_suffix, payload_md5, result):
        """
        This only seems to be relevant for INIT and GET flags currently, but additional info can be given regarding
        the status of the update.

        I think perhaps a new flag should be invented for this, or it should be in the DONE flag where the RESULT
        can have several meanings but what do I know.
        +-----------+----------+-----------------------------------------------------+
        | Extension | Contents |                       Meaning                       |
        +-----------+----------+-----------------------------------------------------+
        | .INIT     | RESULT=0 | Payload is new, update required                     |
        | .INIT     | RESULT=1 | Payload matches previous, no update needed          |
        | .GET      | RESULT=0 | Beginning to download files                         |
        | .GET      | RESULT=1 | Payload too big to be loaded (that's what she said) |
        +-----------+----------+-----------------------------------------------------+

        """
        binary_stream = io.BytesIO()
        flag_contents = "RESULT=%s" % result
        binary_stream.write(flag_contents.encode())

        binary_stream.seek(0)
        return binary_stream


    def check_get_flag(self, log_flag_path, file_md5):
        """
        This needs to check whether a GET flag exists, if it does download it and check its contents.

        This only applies for a destination_list though.  More special treatment for it
        """
        filename = file_md5 + ".GET"

        get_flag_path = os.path.join(log_flag_path, filename)
        local_path = os.path.join("/tmp", filename)
        if not self.ftp.download_file(get_flag_path, local_path):
            with open(local_path, "r") as flag_file:
                contents = flag_file.read()

            if len(contents) > 0:
                logging.info("WDM: GET FLAG not empty, assuming failed previous load")
                return False
            else:
                logging.info("WDM: GET FLAG empty")
                return True

        logging.info("WDM: GET FLAG missing")
        return True

    """
    ###################################################################################################################
    PAYLOAD HANDLING
    """

    def process_downloaded_payloads(self, token_results) -> dict:
        """
        This is the bit that deals with all the updates that have been downloaded, now it is a case of dealing with them
        one by one and moving them to the right place or sending them to the console.

        This takes the dictionary generated from self.process_payload_tokens and looks like this:
        {
            "DB_FILE": {
                "ready": False,
                "md5": "",
                "loaded": False,
                "flag": "",
                "check_pass": False
            }
        }

        It returns the same dictionary with updated fields if appropriate

        Need to figure out how to link console_task with the uploading of files.  Before console task would look for a single
        flag but this would be trickier to achieve as there would be a lot of waiting to be done.

        Internal flags now only changed on success of loading a token.
        """
        _token_results = token_results

        for payload_name, payload_status in _token_results.items():
            payload_attributes = self.obtain_payload_attributes(payload_name)
            token = payload_attributes["token"]
            log_path = payload_attributes["log_path"]
            transfer = payload_attributes["transfer"]
            check_size = payload_attributes.get("check_size", False)

            #check_size only applies to console databases, all other payload types can skip this
            if check_size:
                try:
                    _importer_has_checked = self.importing_module_flags[payload_name]["check_pass"]
                except KeyError:
                    pass
                    # print("WDM: Hm, check_pass value missing for a payload that needs it")
                except TypeError:
                    #For when the importer flags have not been updated yet
                    pass
                else:
                    if _importer_has_checked is True and _token_results[payload_name]["flag"] != "LOAD":
                        self.update_status_flag("GOT", log_path, payload_status["md5"])
                        if self.update_status_flag("LOAD", log_path, payload_status["md5"]):
                            _token_results[payload_name]["flag"] = "LOAD"
                            #No need to check the GOT flag as it will be reloaded again
                    elif _importer_has_checked is False:
                        if self.update_status_flag("GET", log_path, payload_status["md5"], result=1):
                            #Set the flags appropriately so no more action is taken but the rest of the app thinks its complete
                            _token_results[payload_name]["loaded"] = True
                            _token_results[payload_name]["flag"] = "DONE"
                    elif _importer_has_checked == "":
                        logging.info("WDM: Waiting for size check %s" % token)
                    else:
                        pass

            if payload_status["ready"] and payload_status["loaded"] is False:
                logging.info("WDM: processing %s" % token)

                self.set_importer_task_flag(payload_name, payload_status)
                #Set the LOAD flag but we only need to do this once!!  This is applied to all payload types except for
                #controller databases which follows a separate annoying path via check_size above
                if payload_status["flag"] != "LOAD":
                    if not check_size:
                        if self.update_status_flag("LOAD", log_path, payload_status["md5"]):
                            _token_results[payload_name]["flag"] = "LOAD"

                # Moves the file locally and then sets the DONE flag
                if not transfer and payload_status["loaded"] is False:
                    #This part is for local files
                    status = self.move_payload(payload_attributes)
                    if status == 0:
                        #Now only internal flags are updated when the DONE flag has succesully loaded
                        if self.update_status_flag("DONE", log_path, payload_status["md5"]):
                            _token_results[payload_name]["loaded"] = True
                            logging.info("\t Local file loaded")
                            self.update_importer_task_flag(payload_name)
                    elif status == 2:
                        pass
                    else:
                        pass
                else:
                    """This part is all the files that need to go to the console.  We dont do any transferring here, just
                    set a series of flags to let the importing module know but only set it """
                    self.set_importer_task_flag(payload_name, payload_status)

            else:
                try:
                    _importer_has_loaded = self.importing_module_flags[payload_name]["loaded"]
                except KeyError:
                    pass
                except TypeError:
                    pass
                else:
                    logging.debug("WDM: _importer_has_loaded %s %s" % (payload_name, _importer_has_loaded))

                    if _importer_has_loaded and _token_results[payload_name]["flag"] != "DONE":
                        if self.update_status_flag("DONE", log_path, payload_status["md5"]):
                            _token_results[payload_name]["flag"] = "DONE"

        logging.info("Skipping other payloads")
        return _token_results

    def move_payload(self, payload_attributes) -> int:
        """
        This moves the file from /tmp to wherever it needs to be locally
        """
        copy_success = False
        file_name = payload_attributes["file_name"]
        local_path = payload_attributes["file_local_path"]

        if local_path is None:
            #If all the tokens are properly defined then this should never happen.
            return 2

        file_src = os.path.join(self.local_tempoary_path, file_name)
        # local_path = os.path.join(local_path, file_name)        #This is needed so that existing files are overwritten

        ""
        for retries in range(5):
            try:
                shutil.move(file_src, local_path)
            except OSError as e:
                logging.error(f"WDM: Could not move payload {e}")
                continue
            else:
                logging.info(f"WDM: moving payload {file_name} to {local_path} succesful")
                copy_success = True
                break

        if copy_success:
            return 0
        else:
            return 1

    def check_all_payloads_complete(self, token_results) -> int:
        """
        This checks that all the payloads obtained payloads have finished installing and thus can reset the state machine
        """
        _token_results = token_results
        ready_list = []
        loaded_list = []
        updates_ready = False

        for payload, payload_status in _token_results.items():
            ready_list.append(payload_status["ready"])
            loaded_list.append(payload_status["loaded"])

        for index, ready in enumerate(ready_list):
            if ready == True:
                updates_ready = True
                if ready != loaded_list[index]:
                    return 0

        if updates_ready:
            return 1
        else:
            return 2

    """
    ###################################################################################################################
    FORCED UPDATES
    """

    def set_forced_update(self, payloads: list):
        """
        This forces a download to happen regardless of status of the UIDs/Flags

        payloads is a list of items to grab where each element is the key of the payload attributes previously
        supplied to this module
        """
        self.forced_update_required = True
        self.list_of_forced_updates = payloads
        self.forced_update_ready = None
        self.set_destination_list_too_phat(False)

    def reset_force_update(self, all=False):
        """
        This resets the forced update flags
        """
        self.forced_update_required = False
        self.list_of_forced_updates = None
        if all:
            self.forced_update_ready = None
            self.set_destination_list_too_phat(False)

    def set_destination_list_too_phat(self, flag=True):
        """
        This sets the destination list too phat flag
        """
        self.database_too_phat = flag

    def force_download_file(self, tokens_to_process):
        """
        This is the routine that does the downloading, it is simular to the process_payload_tokens but a very stripped down version

        Only really applies to destination databases but whatever.
        """
        forced_payloads_to_process = {}

        for payload in tokens_to_process:
            payload_status = {
                "ready": False,
                "too_big": False
            }

            forced_payloads_to_process[payload] = payload_status
            logging.info("WDM: force processing %s" % payload)
            payload_attributes = self.obtain_payload_attributes(payload)
            token_name = payload_attributes["token"]
            log_path = payload_attributes["log_path"]

            if self.get_token(token_name):
                payload_path, payload_md5, payload_extension = self.parse_token(token_name)
                if payload_path and payload_md5:
                    payload_name = payload_attributes["file_name"]
                    if payload == "destination_list" and self.database_too_phat:
                        if not self.update_status_flag("GET", log_path, payload_md5, result=1):
                            self.database_too_phat = False
                            forced_payloads_to_process[payload]["ready"] = True
                            forced_payloads_to_process[payload]["too_big"] = True
                            continue

                    if self.check_get_flag(log_path, payload_md5):
                        if not self.ftp.download_file(payload_path,
                                                      os.path.join(self.local_tempoary_path, payload_name),
                                                      1):
                            forced_payloads_to_process[payload]["ready"] = True
                    else:
                        forced_payloads_to_process[payload]["ready"] = True
                        forced_payloads_to_process[payload]["too_big"] = True

        logging.debug(json.dumps(forced_payloads_to_process))
        self.forced_update_ready = forced_payloads_to_process

    """
    ###################################################################################################################
    Importer Flags - Calls for importing modules
    """
    def set_importer_task_flag(self, payload, payload_status) -> None:
        """
        This appends a payload to the importing module flag that is to be used by the importing class
        """
        if self.importing_module_flags is None:
            self.importing_module_flags = {}

        #Only append it if it is missing, otherwise just skip
        try:
            self.importing_module_flags[payload]
        except KeyError:
            self.importing_module_flags[payload] = payload_status

    def get_importer_task_flags(self) -> None:
        """
        The intention of this is to allow whatever importing module to check what needs processing and updating
        """
        return self.importing_module_flags

    def update_importer_task_flag(self, payload, loaded=True) -> None:
        """
        The intention of this is to allow whatever importing module to update the flags
        """
        try:
            self.importing_module_flags[payload]["loaded"] = loaded
        except KeyError:
            logging.warning("WDM: Cannot update flag %s" % payload)
        else:
            logging.info("WDM: %s loaded" % payload)

    def update_importer_database_size_check_flag(self, payload, passed) -> None:
        """
        When a payload has a (optional) size check flag set, then this will set the check to either true or false
        """
        try:
            self.importing_module_flags[payload]["check_pass"] = passed
        except KeyError:
            logging.warning("WDM: Cannot update size check flag %s" % payload)
        else:
            logging.info("WDM: %s payload check size pass: %s" % (payload, passed))

    def reset_importer_task_flags(self) -> None:
        """
        This resets the flags to none
        """
        self.importing_module_flags = None

    def get_wdm_status(self):
        """
        This returns the current state of the state machine
        """

        return self.wdm_status

    def set_importer_reset_wait(self, state: bool = True) -> None:
        """
        This is used to tell the state machine to wait for the importer to tell it to reset instead of going it automatically
        THe default state is "Off" but calling this routine without any args will switch it on.
        """
        logging.info("WDM: Auto state machine reset %s" % state)
        self.wait_for_importer_reset = state

    def importer_reset_state_machine(self, reset: bool = True) -> None:
        """
        This allows the importer to reset the state machine if it is dependent on the importer doing so.
        """
        self.importer_reset_requested = reset

        if reset:
            logging.info("WDM: State machine reset requested")

    """
    ###################################################################################################################
    WDM Status HANDLING
    """

    def generate_device_status_file(self):
        """
        Generates the status file as per the description above.  The contained fields have been agreed with Andrew Latham

        The original version stripped out ONION in the firmware version, this has been reinstated.
        :return:
        """
        status_contents = {
            "VARIANT_NAME": self.hw_dict["model"],
            "FIRMWARE_VERSION": self.hw_dict["software_version"],
            "SUPPORTED_PAYLOAD_TYPES": ','.join(self.get_supported_payloads()),
            "ONION_VERSION": self.hw_dict["onion_ver"],
            "UNIT_ID": self.unit_id,
            "CONTROLLER_PROFILE_NUMBER": self.config_dict.get("controller_profile", "N/A"),
            "STATUS": "STATUS_OK",
            "SERIAL_NUMBER": self.hw_dict["serial_number"]
        }

        binary_stream = io.BytesIO()

        for (parameter, value) in status_contents.items():
            temp = "%s = %s\r\n" % (parameter, value)
            binary_stream.write(temp.encode())

        binary_stream.write(self.generate_network_config_values().encode())
        #Insert sign status' here
        if self.sign_statuses != None:
            binary_stream.write(self.process_sign_status())

        binary_stream.write(self.generate_ftp_status_values().encode())

        binary_stream.seek(0)
        return binary_stream

    def process_sign_status(self):
        """
        This converts the sign status list into something WDM understands.  The sign status bit looks like this:
        Sign status :
        SIGN_STATUS=ADDR1="Sign1"=STATUS_OK
        SIGN_STATUS=ADDR4="Sign2"=STATUS_NO_RESP_ERR

        This now also supports sign firmware versions
        Sign firmware :
        SIGN_FIRMWARE=ADDR0=""=OLEMS 1.39.01
        SIGN_FIRMWARE=ADDR1=""=OLED 1.40.01
        :return:
        """
        status_code = {
            "0": "STATUS_OK",
            "1": "STATUS_MESSAGE_CONTENT_ERR",
            "2": "STATUS_CHECKSUM_ERR",
            "3": "STATUS_BULB_ERR",
            "4": "STATUS_NO_RESP_ERR",
            "5": "STATUS_BAD_STATUS_ERR",
            "6": "STATUS_COMMS_ERR"
        }

        status_text = "Sign status :\r\n"

        for address, status in enumerate(self.sign_statuses):
            if status == ".":
                continue

            status_text += "   SIGN_STATUS=ADDR%s=\"\"=%s\r\n" % (address, status_code.get(status, "STATUS_NO_RESP_ERR"))

        if self.sign_firmwares is not None:
            status_text += "Sign firmware :\r\n"

            #Only list firmware for fitted signs
            for address, status in enumerate(self.sign_statuses):
                if status == ".":
                    continue
                else:
                    firmware = self.sign_firmwares[address]
                    status_text += "   SIGN_FIRMWARE=ADDR%s=\"\"=%s\r\n" % (address, firmware)

        return status_text.encode()

    def generate_network_config_values(self):
        """
        This creates the network config values needed in the status file
        """

        network_text = "Addresses from Config :\r\n"
        network_text += "   IP address           = %s\r\n" % self.config_dict.get("ip_address", " ")
        network_text += "   Subnet mask          = %s\r\n" % self.config_dict.get("subnet_mask", " ")
        network_text += "   Gateway address      = %s\r\n" % self.config_dict.get("gateway", " ")

        return network_text

    def generate_ftp_status_values(self):
        """

        """
        ftp_text = "Configurable values (and defaults) :\r\n"
        ftp_text += "   DHCP_SERVER = %s\r\n" % ("1" if self.config_dict.get("dhcp_server", False) else "0")
        ftp_text += "   DHCP_CLIENT = %s\r\n" % ("1" if self.config_dict.get("dhcp_client", False) else "0")
        ftp_text += "   IP_ADDRESS = %s\r\n" % self.config_dict.get("ip_address_configured", "")
        ftp_text += "   SUBNET_MASK = %s\r\n" % self.config_dict.get("subnet_mask", "")
        ftp_text += "   DEFAULT_GATEWAY = %s\r\n" % self.config_dict.get("gateway", "")
        ftp_text += "   FTP_SERVER_IP = %s\r\n" % self.config_dict.get("server_ip", "")
        ftp_text += "   FTP_SERVER_USER_NAME = %s\r\n" % self.config_dict.get("username", "")
        ftp_text += "   FTP_SERVER_PASSWORD = ????\r\n"
        ftp_text += "   FTP_SERVER_DELAY_WAIT = %s\r\n" % self.config_dict.get("holdoff", "")
        ftp_text += "   UNIT_ID = %s\r\n" % self.unit_id
        ftp_text += "   FTP_PORT = %s\r\n" % self.config_dict.get("server_port", "")

        return ftp_text

    def update_device_status(self):
        """
        Uploads the device status to the FTP server.

        Status files should either be called
        :return:
        """
        logging.info("WDM: Updating device status")

        try:
            status_path = self.cloud_paths["status_path"].replace("<UNIT_ID>", self.unit_id)
        except KeyError:
            #Use scheme 1 if its not defined.
            status_path = r"/vehicle/<UNIT_ID>/log".replace("<UNIT-ID>", self.unit_id)

        status_file = self.generate_device_status_file()

        if self.hw_dict["hw_type"] == "con":
            if self.hw_dict["model"].endswith("ONION"):
                model = self.hw_dict["model"][:-5]
            else:
                model = self.hw_dict["model"]
        else:
            model = "SIGN"

        try:
            status_file_path = os.path.join(status_path, model + "-STATUS.txt")
            self.upload_file(status_file_path, status_file)
        except OSError:
            # There is a weird bug where even when a successful flag is sent, it raises an exception Errno 0
            pass
        except ftplib.all_errors as e:
            logging.warning(e)
            logging.warning("\tFailed :(")
            return 0

    """
    ###################################################################################################################
    WDM MAIN
    """

    def reset_state_machine(self) -> None:
        """
        Resets the state machine and all other associated stuff
        """
        self.updates_available = False
        self.importer_reset_requested = False

        if self.wait_for_importer_reset:
            self.reset_importer_task_flags()

    def run(self):
        """
        This is the main loop, it is a good idea to update the wdm parameters before running this but if there is no valid
        unit ID then it will just sit in a loop until it is set.

        One important variable to note is self.allow_download.  When this is False, no tokens, downloads will be processed
        """
        while 1:
            #Sit in a loop until WDM parameters are set otherwise the rest will break.
            if self.unit_id is None:
                logging.info("WDM: Waiting for parameters to be set")
                time.sleep(1)
            else:
                break

        #Obtain the tokens that should be downloaded from the server
        payloads_to_process = {}
        uids_processed = False

        while 1:
            #Connect to FTP server and immediately upload status file
            if self.ftp.connectFTP():
                #Set to true for now, but switch of when its depreciated
                if self.config_dict.get("WDM_enable_ftp_txt_status", True):
                    self.update_device_status()

                if not self.updates_available:
                    if self.allow_download:
                        tokens_to_process = self.check_payload_uid_files(self.get_payload_attributes())
                        if len(tokens_to_process) > 0:
                            logging.info("WDM: Updates available")
                            self.updates_available = True
                            self.reset_importer_task_flags()
                            uids_processed = False
                            #Go and download all the appropriate payloads depending on the parameters of the tokens
                            payloads_to_process = self.process_payload_tokens(tokens_to_process)
                        else:
                            logging.info("WDM: No update to process")
                    else:
                        logging.info("WDM: Waiting for permission to check for updates")
                    # logging.debug("!!!!!! %s %s" %(self.forced_update_required, self.updates_available))
                if self.forced_update_required:
                    logging.info("WDM: Forced updates required")
                    self.force_download_file(self.list_of_forced_updates)

                    #I think it is a fair assumption for this to just download and let whatever has requested the
                    #forced update to look for the file
                    self.reset_force_update()

                if self.updates_available:
                    # Deal with the files that were downloaded
                    payloads_to_process = self.process_downloaded_payloads(payloads_to_process)
                    logging.debug(json.dumps(payloads_to_process))
                    #Check here for the payloads_to_process, if all are processed then we can reset it and the flags
                    status = self.check_all_payloads_complete(payloads_to_process)
                    logging.debug("WDM: check complete status %s" % status)
                    self.wdm_status = status
                    if status == 0:
                        time.sleep(self.timetowait)
                        continue

                    if status == 1:
                        logging.info("WDM: All updates complete")
                        if not uids_processed:
                            if self.update_uids(payloads_to_process):
                                logging.info("WDM: All UIDs sent")
                                uids_processed = True

                    elif status == 2:
                        logging.info("WDM: No updates, resetting state machine")
                        #Always reset regardless
                        self.reset_state_machine()
                        self.reset_importer_task_flags()
                        payloads_to_process = {}

                    # Wait for importer if needed
                    if self.wait_for_importer_reset:
                        if self.importer_reset_requested:
                            self.reset_state_machine()
                            payloads_to_process = {}
                        else:
                            time.sleep(self.timetowait)
                            continue
                    else:
                        self.reset_state_machine()
                        payloads_to_process = {}

            self.ftp.close_ftp()

            if self.break_loop:
                break

            current_ctrl_loading_state = self.allow_download
            for second in range(self.timetowait):
                if not current_ctrl_loading_state:
                    if self.allow_download:
                        break
                #This allows one to get out of the loop if the state of the ctrl_loading changes
                time.sleep(1)

"""
###################################################################################################################
Debug functions
"""
def generate_server_config_file() -> None:
    """
    This generates an empty config file to populate subsequently so that this module can then use it as its
    probably not a good idea to store server details in the source code lol
    """
    empty_config_dict = {
    "server_ip": "sftp://sftp.ver.hanover.cloud",
    "username": "",
    "password": "",
    "server_port": 21,
    "enable_tls": False,
    "unit_id": "0001",
    "holdoff": 5
    }

    print("WDM: Generating blank config file in /tmp")
    print("\t fill in the required details and move to /etc/hanip")
    with open("/tmp/wdm_server_config.cfg", "w", encoding="utf-8") as wdm_file:
        json.dump(empty_config_dict, wdm_file, ensure_ascii=False, indent=4)

def read_config_file():
    """

    """
    if "wdm_server_config.cfg" in os.listdir("/etc/hanip"):
        hwjson = open(os.path.join("/etc/hanip", "wdm_server_config.cfg"), "r").read()

        try:
            jsondata = json.loads(hwjson)
        except json.decoder.JSONDecodeError:
            print("WDM: Can't read server config file")
            return None
        else:
            return jsondata
    else:
        generate_server_config_file()

if __name__ == '__main__':
    import _thread

    logging.basicConfig(level=logging.DEBUG)

    """
    CONSOLE ATTRIBUTES
    """
    cloud_scheme = "1"

    hw_dict_console = {
        "model": "DG3",
        "hw_type": "con",
        "software_version": "1.23.45",
        "serial_number": "0123456789",
        "onion_ver": "2.11.0"
    }

    if cloud_scheme == "1":
        cloud_paths_console = {
            "status_path": r"/vehicle/<UNIT_ID>/log",
            "token_root_path": r"/vehicle/<UNIT_ID>"
        }
    else:
        device = hw_dict_console["model"]
        device_path = device.upper()

        cloud_paths_console = {
            "status_path": r"/vehicle/<UNIT_ID>/<PATH>/log".replace("<PATH>", device_path),
            "token_root_path": r"/vehicle/<UNIT_ID>/<PATH>".replace("<PATH>", device_path)
        }

    payload_types_console = {
        "onion_config": {
            "token": "ONION_CONFIG_FILE",
            "file_name": "config.cfg",
            "file_local_path": "/etc/hanip/config.cfg",
            "transfer": False
        },
        "fontlib": {
            "token": "FONTLIB_FILE",
            "file_name": "fontlib.bin",
            "file_local_path": "/usr/share/renderbox/fontlib.bin",
            "transfer": False
        },
        "console_config": {
            "token": "CONSOLE_CONFIG_FILE",
            "file_name": "config.ini",
            "file_local_path": None,
            "transfer": False
        },
        "console_firmware": {
            "token": "CONSOLE_FIRMWARE_FILE",
            "file_name": "firmware.han",
            "file_local_path": None,
            "transfer": False
        },
        "destination_list": {
            "token": "DB_FILE",
            "file_name": "eric.bin",
            "file_local_path": None,
            "transfer": False,
            "check_size": True
        },
    }

    """
    SIGN ATTRIBUTES
    """
    hw_dict_sign = {
        "model": "G4.018CA.M24.018",
        "hw_type": "ext",
        "address": "0",
        "software_version": "1.23.45",
        "serial_number": "0123456789",
        "onion_ver": "2.11.0"
    }

    cloud_paths_sign = {
        "status_path": r"/vehicle/<UNIT_ID>/<SIGN_ADDRESS>/log".replace("<SIGN_ADDRESS>", hw_dict_sign["address"]),
        "token_root_path": r"/vehicle/<UNIT_ID>/<SIGN_ADDRESS>".replace("<SIGN_ADDRESS>",
                                                                         hw_dict_sign["address"])
    }

    payload_types_sign = {
        "onion_config": {
            "token": "ONION_CONFIG_FILE",
            "file_name": "config.cfg",
            "file_local_path": "/etc/hanip/config.cfg",
            "transfer": False
        },
        "fontlib": {
            "token": "FONTLIB_FILE",
            "file_name": "fontlib.bin",
            "file_local_path": "/usr/share/renderbox/fontlib.bin",
            "transfer": False
        },
        "sign_firmware": {
            "token": "SIGN_FIRMWARE_FILE",
            "file_name": "signfirmware.bin",
            "file_local_path": None,
            "transfer": True
        },
        "onion_firmware": {
            "token": "ONION_FIRMWARE_FILE",
            "file_name": "hanip_update.whl",
            "file_local_path": None,
            "transfer": True  # This is an exception
        },
    }

    if 1:
        hw_dict = hw_dict_console
        payload_types = payload_types_console
        cloud_paths = cloud_paths_console
    else:
        hw_dict = hw_dict_sign
        payload_types = payload_types_sign
        cloud_paths = cloud_paths_sign

    server_details = read_config_file()
    if server_details == None:
        pass
    else:
        wdm = WDMClient(hw_dict, "/tmp")
        wdm.update_wdm_configurations(server_details)
        wdm.setup_ftp_client()
        wdm.set_payload_attributes(payload_types)
        wdm.set_cloud_paths(cloud_paths)
        _thread.start_new_thread(wdm.run, ())

        while 1:
            time.sleep(1)