@file:Suppress("NonAsciiCharacters", "ObjectPropertyName")

package com.lanyuanxiaoyao.digtal.market

import com.lanyuanxiaoyao.squirrel.core.common.Content
import com.lanyuanxiaoyao.squirrel.core.common.Downloader
import com.lanyuanxiaoyao.squirrel.core.common.Parser
import com.lanyuanxiaoyao.squirrel.core.common.Process
import com.lanyuanxiaoyao.squirrel.core.common.Replace
import com.lanyuanxiaoyao.squirrel.core.common.Rule
import com.lanyuanxiaoyao.squirrel.core.common.Script
import com.lanyuanxiaoyao.squirrel.core.common.Selector
import com.lanyuanxiaoyao.squirrel.core.common.Site

private val html = Pair("html", "true")
private val iframe = Pair("iframe", "true")
private val post = Pair("post", "true")

private val timeScript = Script(
    Script.Type.Javascript,
    // language=javascript
    script = "if (text && text !== '') {\n    return text\n} else if(params['datetime']) {\n    return params['datetime']\n} else {\n    return text\n}",
)

private val titleScript = Script(
    Script.Type.Javascript,
    // language=javascript
    script = "if (text && text !== '') {\n    return text\n} else if(params['title']) {\n    return params['title']\n} else {\n    return text\n}",
)

// language=regexp
private val commonRemove = listOf(
    "<.+?>",
    "&.+?;"
)

private val 广东政务服务和数据管理局 = Site(
    code = "74ee6b33-c1a3-41f9-b947-acd0bebd0e6e",
    name = "广东政务服务和数据管理局",
    home = "https://zfsg.gd.gov.cn",
    parser = Parser.Type.CSS,
    author = "lanyuanxiaoyao",
    target = Site.Target.SEARCH,
    downloader = Downloader.Type.BROWSER,
    search = "https://search.gd.gov.cn/api/search/all?json=%7B%22page%22%3A1%2C%22keywords%22%3A%22{query}%22%2C%22advance%22%3A%22true%22%2C%22sort%22%3A%22time%22%2C%22position%22%3A%22all%22%2C%22time_to%22%3A2524579200%2C%22time_from%22%3A189273600%2C%22site_id%22%3A%22246%22%2C%22range%22%3A%22site%22%2C%22recommand%22%3A1%7D",
    rules = mapOf(
        // language=regexp
        "https://search.gd.gov.cn/api/search/all.*" to Rule(
            downloader = Downloader.Type.HTTP,
            parser = Parser.Type.JSON,
            list = Content(
                expression = "$.data.news.list",
                title = Selector("$.title", process = Process(remove = commonRemove)),
                author = Selector("$.source"),
                dateTime = Selector("$.publish_time"),
                link = Selector("$.url"),
            ),
            properties = mapOf(post),
            next = Selector(
                "$.data.news.total", process = Process(
                    script = listOf(
                        Script(
                            Script.Type.Javascript,
                            // language=javascript
                            "let url = params['url']\nif ((url && url !== '') && (text && text !== '')) {\n    let total = parseInt(text)\n    let postData = JSON.parse(params['json'])\n    let current = parseInt(postData['page'] ?? '1')\n    if (current * 20 >= total) {\n        return ''\n    }\n    postData['page'] = current + 1\n    return 'https://search.gd.gov.cn/api/search/all?json=' + encodeURIComponent(JSON.stringify(postData))\n}\nreturn ''"
                        )
                    )
                )
            )
        ),
        // language=regexp
        "https*://zfsg\\.gd\\.gov\\.cn/gkmlpt/content/.*/post_\\d+\\.html.*" to Rule(
            downloader = Downloader.Type.BROWSER,
            text = Content(
                expression = ".content-container",
                title = Selector(".content-box .content h1.title"),
                author = Selector("td.first:contains(发布机构) + td > span"),
                dateTime = Selector("td.second:contains(成文日期) + td > span"),
                content = Selector(".content .article-content"),
                extra = mapOf(
                    "source" to Selector(".content .article-content", properties = mapOf(html))
                ),
            )
        ),
        // language=regexp
        "https*://zfsg\\.gd\\.gov\\.cn/(xxfb|zwgk|ztzl)/.*content/post_\\d+\\.html" to Rule(
            text = Content(
                expression = ".Con",
                title = Selector("h3.zw-title"),
                author = Selector(".zw-info > .ly", process = Process(remove = listOf("来源\\s*:\\s*"))),
                dateTime = Selector(".zw-info > .time", process = Process(remove = listOf("时间\\s*:\\s*"))),
                content = Selector(".zw"),
                extra = mapOf(
                    "source" to Selector(".zw", properties = mapOf(html))
                )
            )
        ),
        // language=regexp
        "https://mp\\.weixin\\.qq\\.com/s/.+" to Rule(
            text = Content(
                expression = "#page-content",
                title = Selector("#activity-name"),
                author = Selector("#js_name"),
                dateTime = Selector("#publish_time"),
                content = Selector("#js_content"),
                extra = mapOf(
                    "source" to Selector("#js_content", properties = mapOf(html))
                )
            )
        ),
        // language=regexp
        "https*://zfsg\\.gd\\.gov\\.cn/hdjlpt/yjzj/answer/.+" to Rule(
            text = Content(
                expression = "#article-container",
                title = Selector("#article-title"),
                author = Selector("#article-source", process = Process(remove = listOf("发布机构："))),
                dateTime = Selector("#article-published_at", process = Process(remove = listOf("发布时间："))),
                content = Selector("#article-content"),
                extra = mapOf(
                    "source" to Selector("#article-content", properties = mapOf(html))
                )
            )
        ),
        // language=regexp
        "https*://zfsg\\.gd\\.gov\\.cn/hdjlpt/live/index.php\\?pid=\\d+" to Rule(
            text = Content(
                expression = "body",
                title = Selector(".title-page .txt > span"),
                author = Selector(process = Process(default = "广东政务服务和数据管理局")),
                dateTime = Selector(process = Process(script = listOf(timeScript))),
                content = Selector(".content"),
                extra = mapOf(
                    "source" to Selector(".content", properties = mapOf(html))
                )
            )
        ),
    )
)

private val 深圳市政务服务和数据管理局 = Site(
    code = "189a0e12-9319-4a4b-bf3b-34a6282e6f68",
    name = "深圳市政务服务和数据管理局",
    home = "https://www.sz.gov.cn/szzsj/gkmlpt/index",
    icon = "https://www.sz.gov.cn/favicon.ico",
    parser = Parser.Type.CSS,
    author = "lanyuanxiaoyao",
    target = Site.Target.TEXT,
    downloader = Downloader.Type.HTTP,
    properties = mapOf(iframe),
    search = "https://search.gd.gov.cn/jsonp/site/755576?callback=getResult&page=1&pagesize=20&text={query}&order=1&position=all",
    rules = mapOf(
        // language=regexp
        "https*://search\\.gd\\.gov\\.cn/jsonp/site/755576\\?callback=getResult&page=\\d+&pagesize=20&text=.+" to Rule(
            parser = Parser.Type.JSON,
            preload = Process(
                script = listOf(
                    Script(
                        Script.Type.Javascript,
                        // language=javascript
                        "let getResult = result => result\nlet results = eval(text)\nreturn JSON.stringify(results)"
                    )
                )
            ),
            list = Content(
                expression = "$.results",
                title = Selector("$.title", process = Process(remove = commonRemove)),
                dateTime = Selector(
                    "$.first_publish_time",
                    process = Process(
                        script = listOf(
                            Script(
                                Script.Type.Javascript,
                                // language=javascript
                                script = "let datetime = new Date(text * 1000)\nreturn `\${datetime.getFullYear()}-\${datetime.getMonth() + 1}-\${datetime.getDate()} \${datetime.getHours()}:\${datetime.getMinutes()}:\${datetime.getSeconds()}`"
                            )
                        )
                    )
                ),
                link = Selector("$.url", process = Process(replace = listOf(Replace("https", "http")))),
            ),
            next = Selector(
                "$.count",
                process = Process(
                    script = listOf(
                        Script(
                            Script.Type.Javascript,
                            // language=javascript
                            script = "let url = params['url']\nif (url && url !== '') {\n    let current = parseInt(params['page'] ?? '1')\n    let total = parseInt(text)\n    if (current * 20 >= total) {\n        return ''\n    }\n    return url.replace(/page=\\d+/, 'page=' + (current + 1))\n}\nreturn ''"
                        )
                    )
                )
            )
        ),
        // language=regexp
        "https*://www\\.sz\\.gov\\.cn/szzsj/gkmlpt/content/\\d+/\\d+/post_\\d+\\.html" to Rule(
            downloader = Downloader.Type.BROWSER,
            text = Content(
                expression = ".content-container",
                title = Selector(".content-box .content h1.title"),
                author = Selector("td.first:contains(发布机构) + td > span"),
                dateTime = Selector("td.second:contains(成文日期) + td > span"),
                content = Selector(".content .article-content"),
                extra = mapOf(
                    "source" to Selector(".content .article-content", properties = mapOf(html))
                ),
            )
        )
    )
)

private val 中华人民共和国中央人民政府 = Site(
    code = "00bfe1f5-7662-4ea5-ada3-6e9dfc19f0be",
    name = "中华人民共和国中央人民政府",
    home = "https://www.gov.cn",
    icon = "https://www.gov.cn/favicon.ico",
    parser = Parser.Type.CSS,
    author = "lanyuanxiaoyao",
    target = Site.Target.TEXT,
    downloader = Downloader.Type.HTTP,
    properties = mapOf(iframe),
    search = "https://sousuoht.www.gov.cn/athena/forward/2B22E8E39E850E17F95A016A74FCB6B673336FA8B6FEC0E2955907EF9AEE06BE?json=%7B%22code%22%3A%2217da70961a7%22%2C%22historySearchWords%22%3A%5B%5D%2C%22dataTypeId%22%3A%22107%22%2C%22orderBy%22%3A%22time%22%2C%22searchBy%22%3A%22title%22%2C%22appendixType%22%3A%22%22%2C%22granularity%22%3A%22ALL%22%2C%22trackTotalHits%22%3Atrue%2C%22beginDateTime%22%3A%22%22%2C%22endDateTime%22%3A%22%22%2C%22isSearchForced%22%3A0%2C%22filters%22%3A%5B%5D%2C%22pageNo%22%3A1%2C%22pageSize%22%3A10%2C%22customFilter%22%3A%7B%22operator%22%3A%22and%22%2C%22properties%22%3A%5B%5D%7D%2C%22searchWord%22%3A%22{query}%22%7D",
    rules = mapOf(
        // language=regexp
        "https://sousuoht\\.www\\.gov\\.cn/athena/forward/2B22E8E39E850E17F95A016A74FCB6B673336FA8B6FEC0E2955907EF9AEE06BE.+" to Rule(
            parser = Parser.Type.JSON,
            headers = mapOf(
                "Content-Type" to "application/json",
                "Athenaappkey" to "be5qL61qyHKQ87BVlqy%2BwloIkpcL1tCqC4yOM%2F%2BsqaSn0NxuA4GPOWocQu96h0PUaYBe9hz3DVvjGTECMNNou6cjAlUc3UoJlviQAipc4Ha8qxmIeS1sPqdGQ3Unm49j%2BsYN5T%2BzjqKxdA2yz5qLcG5wjZR7rqYVgT98NasByMA%3D",
                "Athenaappname" to "%E5%9B%BD%E7%BD%91%E6%90%9C%E7%B4%A2",
            ),
            properties = mapOf("post" to "true"),
            list = Content(
                expression = "$.result.data.middle.list",
                title = Selector("$.title_no_tag"),
                dateTime = Selector("$.time"),
                link = Selector("$.url"),
            ),
            next = Selector(
                expression = "$.result.data.pager",
                process = Process(
                    script = listOf(
                        Script(
                            Script.Type.Javascript,
                            // language=javascript
                            script = "let query = {\n    code: '17da70961a7',\n    historySearchWords: [],\n    dataTypeId: '107',\n    orderBy: 'time',\n    searchBy: 'title',\n    appendixType: '',\n    granularity: 'ALL',\n    trackTotalHits: true,\n    beginDateTime: '',\n    endDateTime: '',\n    isSearchForced: 0,\n    filters: [],\n    pageNo: 1,\n    pageSize: 10,\n    customFilter: {\n        operator: 'and',\n        properties: []\n    },\n    searchWord: '数据要素'\n}\nif (text && text !== '') {\n    let pager = JSON.parse(text)\n    let current = pager['pageNo']\n    let count = pager['pageCount']\n    if (current < count) {\n        query.pageNo = current + 1\n    } else {\n        return ''\n    }\n}\nreturn `https://sousuoht.www.gov.cn/athena/forward/2B22E8E39E850E17F95A016A74FCB6B673336FA8B6FEC0E2955907EF9AEE06BE?json=\${encodeURIComponent(JSON.stringify(query))}`\n"
                        )
                    )
                )
            )
        ),
        // language=regexp
        "https*://www\\.gov\\.cn/.+/content_\\d+\\.html*" to Rule(
            downloader = Downloader.Type.BROWSER,
            text = Content(
                expression = ".content .article",
                title = Selector("h1#ti", process = Process(script = listOf(titleScript))),
                author = Selector(
                    ".pages-date > .font",
                    process = Process(
                        default = "中华人民共和国中央人民政府",
                        remove = listOf("来源：")
                    )
                ),
                dateTime = Selector(
                    ".pages-date",
                    process = Process(script = listOf(timeScript)),
                    properties = mapOf("precision" to "true")
                ),
                content = Selector(".pages_content"),
                extra = mapOf(
                    "source" to Selector(".pages_content", properties = mapOf("html" to "true"))
                )
            )
        ),
        // language=regexp
        "https://xcx\\.www\\.gov\\.cn/robot/gwypolicy/#/qaReadDetails.+" to Rule(
            downloader = Downloader.Type.BROWSER,
            text = Content(
                expression = ".main-content",
                title = Selector(".qa_content_box", process = Process(script = listOf(titleScript))),
                author = Selector(".qa-subtext-item:contains(来源)", process = Process(remove = listOf("来\\s*源："))),
                dateTime = Selector(process = Process(script = listOf(timeScript))),
                content = Selector(".qa_content_text"),
                extra = mapOf(
                    "source" to Selector(".main-content", properties = mapOf("html" to "true"))
                )
            )
        ),
    )
)

private val 中华人民共和国国家互联网信息办公室 = Site(
    code = "1df28c35-1e9e-4d58-9595-f08029b160b4",
    name = "中华人民共和国国家互联网信息办公室",
    home = "https://www.cac.gov.cn",
    icon = "https://www.cac.gov.cn/favicon.ico",
    parser = Parser.Type.CSS,
    author = "lanyuanxiaoyao",
    target = Site.Target.TEXT,
    downloader = Downloader.Type.HTTP,
    search = "https://search.cac.gov.cn/cms/cmsadmin/infopub/gjjs.jsp?templetid=1563339473064626&pubtype=S&pubpath=portal&page=1&webappcode=A09&huopro={query}&mustpro=&notpro=&inpro=&startDate=\$\$\$&endDate=\$\$\$&sort=1&searchfield=\$\$\$&searchdir=A09",
    rules = mapOf(
        "https://search\\.cac\\.gov\\.cn/cms/cmsadmin/infopub/gjjs\\.jsp\\?templetid=1563339473064626&pubtype=S&pubpath=portal&page=\\d*&webappcode=A09&huopro=.+&mustpro=&notpro=&inpro=&startDate=\\\$*&endDate=\\\$*&sort=1&searchfield=\\\$*&searchdir=A09" to Rule(
            list = Content(
                expression = ".xpage-container .list-item",
                title = Selector("a", process = Process(remove = listOf("\\s*»\\s*"))),
                dateTime = Selector(".search_time"),
                link = Selector("a", "href", process = Process(prefix = "https:"))
            ),
            next = Selector(
                ".xpage-pagination .xpage-pagination-next a:contains(下一页)",
                "href",
                Process(prefix = "https://search.cac.gov.cn/cms/cmsadmin/infopub/")
            )
        ),
        "https://www\\.cac\\.gov\\.cn/.+/c_\\d+\\.htm" to Rule(
            text = Content(
                expression = ".main",
                title = Selector("h1.title"),
                author = Selector("#source", process = Process(remove = listOf("来源："))),
                dateTime = Selector("#pubtime"),
                content = Selector(".main-content"),
                extra = mapOf(
                    "source" to Selector(".main", properties = mapOf("html" to "true"))
                )
            )
        )
    )
)

val 广州数据交易所 = Site(
    code = "9a7f1d8f-4f39-4120-adeb-7435339b97bb",
    name = "广州数据交易所",
    author = "lanyuanxiaoyao",
    home = "https://www.cantonde.com",
    icon = "https://www.cantonde.com/favicon.ico",
    description = "广州数据交易所是广东省深入贯彻落实党中央、国务院关于加快培育数据要素市场，助力数字经济高质量发展工作部署，高标准建设的新型数据交易场所。旨在为市场主体提供合规安全、集约高效的数据流通交易综合性服务。广州数据交易所作为广东省数据要素市场体系的核心枢纽，是畅通数据要素大循环的关键举措，也是推进数据要素市场化配置改革的重要载体。",
    target = Site.Target.SEARCH,
    parser = Parser.Type.JSON,
    search = "https://www.cantonde.com/si/common/searchInfo?json=%7B%22NAME%22%3A%22{query}%22%2C%22IN_CATEGORY%22%3A%22%22%2C%22NOT_IN_CATEGORY%22%3A%22%22%2C%22CATEGORY%22%3A%22%22%2C%22pageNo%22%3A1%2C%22pageSize%22%3A10%7D",
    rules = mapOf(
        // language=regexp
        "https://www\\.cantonde\\.com/si/common/searchInfo\\?json=.+" to Rule(
            list = Content(
                expression = "$.data",
                title = Selector("$.TITLE"),
                dateTime = Selector("$.PUBTIME_ORDER"),
                link = Selector(
                    "$.ID", process = Process(
                        script = listOf(
                            Script(
                                Script.Type.Javascript,
                                // language=javascript
                                "return `https://www.cantonde.com/si/info/detail?json=%7B%22id%22%3A%22\${text}%22%7D`",
                            )
                        )
                    )
                )
            ),
            properties = mapOf(post),
            next = Selector(
                "$.extra.total", process = Process(
                    script = listOf(
                        Script(
                            Script.Type.Javascript,
                            // language=javascript
                            "if (text && text !== '') {\n    let total = parseInt(text)\n    let postData = JSON.parse(params['json'])\n    let current = postData['pageNo']\n    let size = postData['pageSize']\n    if (current * size >= total) {\n        return ''\n    }\n    postData['pageNo'] = current + 1\n    return `https://www.cantonde.com/si/common/searchInfo?json=` + encodeURIComponent(JSON.stringify(postData))\n}\nreturn ''"
                        )
                    )
                )
            ),
        ),
        // language=regexp
        "https://www\\.cantonde\\.com/si/info/detail\\?json=.+" to Rule(
            text = Content(
                expression = "$.data",
                title = Selector("$.TITLE"),
                content = Selector("$.CONTENT", process = Process(remove = commonRemove)),
                dateTime = Selector(
                    process = Process(
                        script = listOf(
                            Script(
                                Script.Type.Javascript,
                                // language=javascript
                                "let datetimeText = params['datetime']\nif (datetimeText && datetimeText !== '') {\n    let datetime = new Date(parseInt(params['datetime']))\n    return `\${datetime.getFullYear()}-\${datetime.getMonth() + 1}-\${datetime.getDate()} \${datetime.getHours()}:\${datetime.getMinutes()}:\${datetime.getSeconds()}`\n}\nreturn ''",
                            )
                        )
                    )
                ),
                author = Selector(process = Process(default = "广州数据交易所"))
            ),
            properties = mapOf(post),
        ),
    ),
)

val sites = setOf(
    广东政务服务和数据管理局,
    深圳市政务服务和数据管理局,
    中华人民共和国中央人民政府,
    中华人民共和国国家互联网信息办公室,
    广州数据交易所,
)

val keywords = setOf(
    "数据要素",
    "数据安全",
    "数据跨境",
    "数据交易",
    "数据基础设施",
    "国家数据局",
)
