WDwdataFtp.mjs

import fs from 'fs'
import get from 'lodash-es/get.js'
import size from 'lodash-es/size.js'
import each from 'lodash-es/each.js'
import map from 'lodash-es/map.js'
import cloneDeep from 'lodash-es/cloneDeep.js'
import isbol from 'wsemi/src/isbol.mjs'
import isestr from 'wsemi/src/isestr.mjs'
import isp0int from 'wsemi/src/isp0int.mjs'
import isfun from 'wsemi/src/isfun.mjs'
import ispm from 'wsemi/src/ispm.mjs'
import cdbl from 'wsemi/src/cdbl.mjs'
import pmSeries from 'wsemi/src/pmSeries.mjs'
import getErrorMessage from 'wsemi/src/getErrorMessage.mjs'
import fsIsFolder from 'wsemi/src/fsIsFolder.mjs'
import fsCopyFile from 'wsemi/src/fsCopyFile.mjs'
import fsCleanFolder from 'wsemi/src/fsCleanFolder.mjs'
import fsCreateFolder from 'wsemi/src/fsCreateFolder.mjs'
import fsDeleteFolder from 'wsemi/src/fsDeleteFolder.mjs'
import fsSyncFolder from 'wsemi/src/fsSyncFolder.mjs'
import fsTreeFolderWithHash from 'wsemi/src/fsTreeFolderWithHash.mjs'
import fsGetFileBasicHash from 'wsemi/src/fsGetFileBasicHash.mjs'
import WDwdataBuilder from 'w-dwdata-builder/src/WDwdataBuilder.mjs'
import downloadFiles from './downloadFiles.mjs'


/**
 * 基於檔案之下載FTP數據與任務建構器
 *
 * 執行階段最新hash數據放置於fdDwAttime，前次hash數據會於結束前自動備份至fdDwCurrent
 *
 * 執行階段最新數據放置於fdDwStorageTemp，前次數據放置於fdDwStorage，於結束前會將fdDwStorage清空，將fdDwStorageTemp複製至fdDwStorage
 *
 * @param {String} st 輸入設定FTP連線資訊物件
 * @param {String} [st.transportation='FTP'] 輸入傳輸協定字串，可選'FTP'、'SFTP'，預設'FTP'
 * @param {String} [st.hostname=''] 輸入hostname字串，預設''
 * @param {Integer} [st.port=21|22] 輸入port正整數，當transportation='FTP'預設21，當transportation='SFTP'預設22
 * @param {String} [st.username=''] 輸入帳號字串，預設''
 * @param {String} [st.password=''] 輸入密碼字串，預設''
 * @param {String} [st.fdIni='./'] 輸入同步資料夾字串，預設'./'
 * @param {Object} [opt={}] 輸入設定物件，預設{}
 * @param {Boolean} [opt.useExpandOnOldFiles=false] 輸入來源檔案是否僅為增量檔案布林值，預設false
 * @param {Boolean} [opt.useSimulateFiles=false] 輸入是否使用模擬取得FTP數據布林值，預設false
 * @param {String} [opt.fdTagRemove='./_tagRemove'] 輸入暫存標記為刪除數據資料夾字串，預設'./_tagRemove'
 * @param {String} [opt.fdDwStorageTemp='./_dwStorageTemp'] 輸入最新下載檔案存放資料夾字串，預設'./_dwStorageTemp'
 * @param {String} [opt.fdDwStorage='./_dwStorage'] 輸入合併儲存檔案資料夾字串，預設'./_dwStorage'
 * @param {String} [opt.fdDwAttime='./_dwAttime'] 輸入當前下載供比對hash用之數據資料夾字串，預設'./_dwAttime'
 * @param {String} [opt.fdDwCurrent='./_dwCurrent'] 輸入已下載供比對hash用之數據資料夾字串，預設'./_dwCurrent'
 * @param {String} [opt.fdResult=`./_result`] 輸入已下載數據所連動生成數據資料夾字串，預設`./_result`
 * @param {String} [opt.fdTaskCpActualSrc='./_taskCpActualSrc'] 輸入任務狀態之來源端完整資料夾字串，預設'./_taskCpActualSrc'
 * @param {String} [opt.fdTaskCpSrc='./_taskCpSrc'] 輸入任務狀態之來源端資料夾字串，預設'./_taskCpSrc'
 * @param {String} [opt.fdLog='./_logs'] 輸入儲存log資料夾字串，預設'./_logs'
 * @param {Function} [opt.funDownload=null] 輸入取得最新下載檔案hash之函數，回傳資料陣列，預設null
 * @param {Function} [opt.funGetCurrent=null] 輸入取得已下載檔案hash之函數，回傳資料陣列，預設null
 * @param {Function} [opt.funAdd=null] 輸入當有新資料時，需要連動處理之函數，預設null
 * @param {Function} [opt.funModify=null] 輸入當有資料需更新時，需要連動處理之函數，預設null
 * @param {Function} [opt.funRemove=null] 輸入當有資料需刪除時，需要連動處理之函數，預設null
 * @param {Function} [opt.funAfterStart=null] 輸入偵測程序剛開始啟動時，需要處理之函數，預設null
 * @param {Function} [opt.funBeforeEnd=null] 輸入偵測程序要結束前，需要處理之函數，預設null
 * @param {Number} [opt.timeToleranceRemove=0] 輸入刪除任務之防抖時長，單位ms，預設0，代表不使用
 * @returns {Object} 回傳事件物件，可呼叫函數on監聽change事件
 * @example
 *
 * import w from 'wsemi'
 * import WDwdataFtp from './src/WDwdataFtp.mjs'
 *
 * let st = {
 *     'hostname': '{hostname}',
 *     'port': 21,
 *     'username': '{username}',
 *     'password': '{password}',
 *     'fdIni': './'
 * }
 * // console.log('st', st)
 *
 * //fdTagRemove
 * let fdTagRemove = `./_tagRemove`
 * w.fsCleanFolder(fdTagRemove)
 *
 * //fdDwStorageTemp
 * let fdDwStorageTemp = `./_dwStorageTemp`
 * w.fsCleanFolder(fdDwStorageTemp)
 *
 * //fdDwStorage
 * let fdDwStorage = `./_dwStorage`
 * w.fsCleanFolder(fdDwStorage)
 *
 * //fdDwAttime
 * let fdDwAttime = `./_dwAttime`
 * w.fsCleanFolder(fdDwAttime)
 *
 * //fdDwCurrent
 * let fdDwCurrent = `./_dwCurrent`
 * w.fsCleanFolder(fdDwCurrent)
 *
 * //fdResult
 * let fdResult = `./_result`
 * w.fsCleanFolder(fdResult)
 *
 * //fdTaskCpActualSrc
 * let fdTaskCpActualSrc = `./_taskCpActualSrc`
 * w.fsCleanFolder(fdTaskCpActualSrc)
 *
 * //fdTaskCpSrc
 * let fdTaskCpSrc = `./_taskCpSrc`
 * w.fsCleanFolder(fdTaskCpSrc)
 *
 * let opt = {
 *     useExpandOnOldFiles: false, //true, false
 *     fdTagRemove,
 *     fdDwStorageTemp,
 *     fdDwStorage,
 *     fdDwAttime,
 *     fdDwCurrent,
 *     fdResult,
 *     fdTaskCpActualSrc,
 *     fdTaskCpSrc,
 *     // fdLog,
 *     // funDownload,
 *     // funGetCurrent,
 *     // funRemove,
 *     // funAdd,
 *     // funModify,
 * }
 * let ev = await WDwdataFtp(st, opt)
 *     .catch((err) => {
 *         console.log(err)
 *     })
 * ev.on('change', (msg) => {
 *     delete msg.type
 *     console.log('change', msg)
 * })
 * // change { event: 'start', msg: 'running...' }
 * // change { event: 'proc-callfun-afterStart', msg: 'start...' }
 * // change { event: 'proc-callfun-afterStart', msg: 'done' }
 * // change { event: 'proc-callfun-download', msg: 'start...' }
 * // change { event: 'proc-callfun-download', num: 2, msg: 'done' }
 * // change { event: 'proc-callfun-getCurrent', msg: 'start...' }
 * // change { event: 'proc-callfun-getCurrent', num: 0, msg: 'done' }
 * // change { event: 'proc-compare', msg: 'start...' }
 * // change { event: 'proc-compare', numRemove: 0, numAdd: 2, numModify: 0, numSame: 0, msg: 'done' }
 * // change { event: 'proc-add-callfun-add', id: 'test1.txt', msg: 'start...' }
 * // change { event: 'proc-add-callfun-add', id: 'test1.txt', msg: 'done' }
 * // change { event: 'proc-add-callfun-add', id: 'test2.txt', msg: 'start...' }
 * // change { event: 'proc-add-callfun-add', id: 'test2.txt', msg: 'done' }
 * // ...
 *
 */
let WDwdataFtp = async(st, opt = {}) => {

    //useExpandOnOldFiles
    let useExpandOnOldFiles = get(opt, 'useExpandOnOldFiles')
    if (!isbol(useExpandOnOldFiles)) {
        useExpandOnOldFiles = false
    }

    //useSimulateFiles, 檔案得預先給予至fdDwStorageTemp
    let useSimulateFiles = get(opt, 'useSimulateFiles')
    if (!isbol(useSimulateFiles)) {
        useSimulateFiles = false
    }

    //fdTagRemove
    let fdTagRemove = get(opt, 'fdTagRemove')
    if (!isestr(fdTagRemove)) {
        fdTagRemove = `./_tagRemove`
    }

    //fdDwStorageTemp, 最新下載檔案存放資料夾
    let fdDwStorageTemp = get(opt, 'fdDwStorageTemp')
    if (!isestr(fdDwStorageTemp)) {
        fdDwStorageTemp = `./_dwStorageTemp`
    }
    if (!fsIsFolder(fdDwStorageTemp)) {
        fsCreateFolder(fdDwStorageTemp)
    }

    //fdDwStorage, 合併儲存檔案資料夾
    let fdDwStorage = get(opt, 'fdDwStorage')
    if (!isestr(fdDwStorage)) {
        fdDwStorage = `./_dwStorage`
    }
    if (!fsIsFolder(fdDwStorage)) {
        fsCreateFolder(fdDwStorage)
    }

    //fdDwAttime
    let fdDwAttime = get(opt, 'fdDwAttime')
    if (!isestr(fdDwAttime)) {
        fdDwAttime = `./_dwAttime`
    }
    if (!fsIsFolder(fdDwAttime)) {
        fsCreateFolder(fdDwAttime)
    }

    //fdDwCurrent
    let fdDwCurrent = get(opt, 'fdDwCurrent')
    if (!isestr(fdDwCurrent)) {
        fdDwCurrent = `./_dwCurrent`
    }
    if (!fsIsFolder(fdDwCurrent)) {
        fsCreateFolder(fdDwCurrent)
    }

    //fdResult
    let fdResult = get(opt, 'fdResult')
    if (!isestr(fdResult)) {
        fdResult = `./_result`
    }
    if (!fsIsFolder(fdResult)) {
        fsCreateFolder(fdResult)
    }

    //fdTaskCpActualSrc
    let fdTaskCpActualSrc = get(opt, 'fdTaskCpActualSrc')
    if (!isestr(fdTaskCpActualSrc)) {
        fdTaskCpActualSrc = `./_taskCpActualSrc`
    }
    if (!fsIsFolder(fdTaskCpActualSrc)) {
        fsCreateFolder(fdTaskCpActualSrc)
    }

    //fdTaskCpSrc
    let fdTaskCpSrc = get(opt, 'fdTaskCpSrc')
    if (!isestr(fdTaskCpSrc)) {
        fdTaskCpSrc = './_taskCpSrc'
    }
    if (!fsIsFolder(fdTaskCpSrc)) {
        fsCreateFolder(fdTaskCpSrc)
    }

    //fdLog
    let fdLog = get(opt, 'fdLog')
    if (!isestr(fdLog)) {
        fdLog = './_logs'
    }
    if (!fsIsFolder(fdLog)) {
        fsCreateFolder(fdLog)
    }

    //funDownload
    let funDownload = get(opt, 'funDownload')

    //funGetCurrent
    let funGetCurrent = get(opt, 'funGetCurrent')

    //funAdd
    let funAdd = get(opt, 'funAdd')

    //funModify
    let funModify = get(opt, 'funModify')

    //funRemove
    let funRemove = get(opt, 'funRemove')

    //funAfterStartCall
    let funAfterStartCall = get(opt, 'funAfterStart')

    //funBeforeEndCall
    let funBeforeEndCall = get(opt, 'funBeforeEnd')

    //timeToleranceRemove
    let timeToleranceRemove = get(opt, 'timeToleranceRemove')
    if (!isp0int(timeToleranceRemove)) {
        timeToleranceRemove = 0
    }
    timeToleranceRemove = cdbl(timeToleranceRemove)

    //getFilesHash
    let getFilesHash = async(vfps) => {

        //ltdtHash
        let ltdtHash = await pmSeries(vfps, async(v) => {
            let id = v.name //用檔名做id
            let hash = await fsGetFileBasicHash(v.path, { type: 'md5' })
            return {
                id,
                hash,
            }
        })

        return ltdtHash
    }

    //treeFolderAndGetFilesHash
    let treeFolderAndGetFilesHash = async (fd) => {

        //vfps
        let vfps = await fsTreeFolderWithHash(fd, 1, { type: 'md5', forFile: true, forFolder: false })

        //ltdtHash
        let ltdtHash = map(vfps, (v) => {
            let id = v.name //用檔名做id
            let hash = v.hash
            return {
                id,
                hash,
            }
        })

        return ltdtHash
    }

    //cvLtdtToKp
    let cvLtdtToKp = (ltdt) => {
        let kp = {}
        each(ltdt, (v) => {
            kp[v.id] = v.hash
        })
        return kp
    }

    //mergeLtdt
    let mergeLtdt = (ltdtNew, ltdtOld) => {
        let kpNew = cvLtdtToKp(ltdtNew)
        let kpOld = cvLtdtToKp(ltdtOld)
        let kp = cloneDeep(kpOld)
        each(kpNew, (hash, id) => {
            kp[id] = hash
        })
        let ltdt = []
        each(kp, (hash, id) => {
            ltdt.push({
                id,
                hash,
            })
        })
        return ltdt
    }

    //vfpsDw
    let vfpsDw = []

    //funDownloadDef
    let funDownloadDef = async() => {

        //vfpsDw, 為下載所得新增檔案清單, 檔案放置於fdDwStorageTemp內
        vfpsDw = await downloadFiles(st, fdDwStorageTemp, {
            useExpandOnOldFiles,
            useSimulateFiles,
        })
        // console.log('vfpsDw', vfpsDw[0], size(vfpsDw))

        //check, 無檔案須報錯, 底層會中止與計算hash與計算檔案hash差異
        if (size(vfpsDw) === 0) {
            throw new Error(`no files`)
        }

        //ltdtHashNewTemp, 計算新增檔案hash值
        let ltdtHashNewTemp = await getFilesHash(vfpsDw)

        //ltdtHashNew
        let ltdtHashNew = []
        if (useExpandOnOldFiles) {

            //ltdtHashOld, 數據來源為fdDwStorage, 為舊hash清單
            let ltdtHashOld = await treeFolderAndGetFilesHash(fdDwStorage)

            //最新合併後檔案hash值清單
            ltdtHashNew = mergeLtdt(ltdtHashNewTemp, ltdtHashOld)

        }
        else {

            //當前下載檔案為全部檔案, 各檔案計算hash值皆須為新hash
            ltdtHashNew = ltdtHashNewTemp

        }
        // console.log('ltdtHashNew', ltdtHashNew)

        //清空fdDwAttime
        fsCleanFolder(fdDwAttime)

        //儲存新hash檔案至fdDwAttime
        each(ltdtHashNew, (v) => {

            //fp
            let fp = `${fdDwAttime}/${v.id}.json` //v.id雖為檔名但視為id使用, fdDwAttime與fdDwCurrent內檔案皆為對應hash檔案, 副檔名為.json

            //writeFileSync
            fs.writeFileSync(fp, JSON.stringify(v), 'utf8')

        })

        return ltdtHashNew
    }
    if (!isfun(funDownload)) {
        funDownload = funDownloadDef
    }

    //funGetCurrentDef
    let funGetCurrentDef = async() => {

        //ltdtHashOld, 數據來源為fdDwStorage, 為舊hash清單
        let ltdtHashOld = await treeFolderAndGetFilesHash(fdDwStorage)

        return ltdtHashOld
    }
    if (!isfun(funGetCurrent)) {
        funGetCurrent = funGetCurrentDef
    }

    //funRemoveDef
    let funRemoveDef = async(v) => {

        let fd = `${fdResult}/${v.id}`
        if (fsIsFolder(fd)) {

            let r = fsDeleteFolder(fd)

            if (r.error) {
                throw new Error(r.error)
            }

        }

    }
    if (!isfun(funRemove)) {
        funRemove = funRemoveDef
    }

    //funAddDef
    let funAddDef = async(v) => {

        let fd = `${fdResult}/${v.id}`
        if (!fsIsFolder(fd)) {
            fsCreateFolder(fd)
        }
        fsCleanFolder(fd)

        let fpStorage = `${fdDwStorageTemp}/${v.id}` //fdDwStorageTemp為新下載檔案, 使用v.id為實際檔案名稱
        let fpResult = `${fd}/${v.id}`
        let r = fsCopyFile(fpStorage, fpResult)

        if (r.error) {
            throw new Error(r.error)
        }

    }
    if (!isfun(funAdd)) {
        funAdd = funAddDef
    }

    //funModifyDef
    let funModifyDef = async(v) => {

        let fd = `${fdResult}/${v.id}`
        if (!fsIsFolder(fd)) {
            fsCreateFolder(fd)
        }
        fsCleanFolder(fd)

        let fpStorage = `${fdDwStorageTemp}/${v.id}` //fdDwStorageTemp為新下載檔案, 使用v.id為實際檔案名稱
        let fpResult = `${fd}/${v.id}`
        let r = fsCopyFile(fpStorage, fpResult)

        if (r.error) {
            throw new Error(r.error)
        }

    }
    if (!isfun(funModify)) {
        funModify = funModifyDef
    }

    //funBeforeEndNec
    let funBeforeEndNec = async() => {

        try {
            srlog.info({ event: 'move-files-to-storage', msg: 'start...' })

            //check, 無檔案須報錯, 底層會中止與計算hash與計算檔案hash差異
            if (size(vfpsDw) === 0) {
                throw new Error(`no files`)
            }

            //useExpandOnOldFiles
            if (useExpandOnOldFiles) {
                //增量模式, 僅將新下載檔案儲存至fdDwStorage

                //複製fdDwStorageTemp內新下載檔案至合併儲存資料夾fdDwStorage
                each(vfpsDw, (v) => {

                    //fsCopyFile
                    let fpSrc = v.path
                    let fpTar = `${fdDwStorage}/${v.name}`
                    let r = fsCopyFile(fpSrc, fpTar)

                    //check
                    if (r.error) {
                        throw new Error(r.error)
                    }

                })

            }
            else {
                //全量模式, 將新下載檔案視為全部檔案並儲存至fdDwStorage

                //fsSyncFolder, 將fdDwStorageTemp完全同步至fdDwStorage
                await fsSyncFolder(fdDwStorageTemp, fdDwStorage)

            }

            //清空fdDwStorageTemp
            fsCleanFolder(fdDwStorageTemp)

            srlog.info({ event: 'move-files-to-storage', msg: 'done' })
        }
        catch (err) {
            console.log(err)
            srlog.error({ event: 'move-files-to-storage', msg: getErrorMessage(err) })
        }

    }

    let funAfterStart = async() => {

        if (isfun(funAfterStartCall)) {
            let r = funAfterStartCall()
            if (ispm(r)) {
                r = await r
            }
        }

        //無funAfterStartNec

    }

    let funBeforeEnd = async() => {

        await funBeforeEndNec()

        if (isfun(funBeforeEndCall)) {
            let r = funBeforeEndCall()
            if (ispm(r)) {
                r = await r
            }
        }

    }

    let optBdr = {
        fdTagRemove,
        fdDwAttime,
        fdDwCurrent,
        fdResult,
        fdTaskCpActualSrc,
        fdTaskCpSrc,
        fdLog,
        funDownload,
        funGetCurrent,
        funRemove,
        funAdd,
        funModify,
        funAfterStart,
        funBeforeEnd,
        timeToleranceRemove,
    }
    let ev = await WDwdataBuilder(optBdr)

    //srlog
    let srlog = ev.srlog

    return ev
}


export default WDwdataFtp