Web Scraper + Elasticsearch + Kibana + SearchKit 打造的豆瓣电影top250 搜索演示系统

作者：小森同学

声明：电影数据来源于“豆瓣电影”，如有侵权，请联系删除

Web Scraper

{
    "_id": "top250",
    "startUrl": ["https://movie.douban.com/top250?start=[0-225:25]&filter="],
    "selectors": [{
        "id": "container",
        "multiple": true,
        "parentSelectors": ["_root"],
        "selector": ".grid_view li",
        "type": "SelectorElement"
    }, {
        "id": "name",
        "multiple": false,
        "parentSelectors": ["container"],
        "regex": "",
        "selector": "span.title:nth-of-type(1)",
        "type": "SelectorText"
    }, {
        "id": "number",
        "multiple": false,
        "parentSelectors": ["container"],
        "regex": "",
        "selector": "em",
        "type": "SelectorText"
    }, {
        "id": "score",
        "multiple": false,
        "parentSelectors": ["container"],
        "regex": "",
        "selector": "span.rating_num",
        "type": "SelectorText"
    }, {
        "id": "review",
        "multiple": false,
        "parentSelectors": ["container"],
        "regex": "",
        "selector": "span.inq",
        "type": "SelectorText"
    }, {
        "id": "year",
        "multiple": false,
        "parentSelectors": ["container"],
        "regex": "\\d{4}",
        "selector": "p:nth-of-type(1)",
        "type": "SelectorText"
    }, {
        "id": "tour_guide",
        "multiple": false,
        "parentSelectors": ["container"],
        "regex": "^导演: \\S*",
        "selector": "p:nth-of-type(1)",
        "type": "SelectorText"
    }, {
        "id": "type",
        "multiple": false,
        "parentSelectors": ["container"],
        "regex": "[^/]+$",
        "selector": "p:nth-of-type(1)",
        "type": "SelectorText"
    }, {
        "id": "area",
        "multiple": false,
        "parentSelectors": ["container"],
        "regex": "[^\\/]+(?=\\/[^\\/]*$)",
        "selector": "p:nth-of-type(1)",
        "type": "SelectorText"
    }, {
        "id": "detail_link",
        "multiple": false,
        "parentSelectors": ["container"],
        "selector": ".hd a",
        "type": "SelectorLink"
    }, {
        "id": "director",
        "multiple": false,
        "parentSelectors": ["detail_link"],
        "regex": "",
        "selector": "span:nth-of-type(1) .attrs a",
        "type": "SelectorText"
    }, {
        "id": "screenwriter",
        "multiple": false,
        "parentSelectors": ["detail_link"],
        "regex": "(?<=编剧: )[\\u4e00-\\u9fa5A-Za-z0-9/()\\·\\s]+(?=主演)",
        "selector": "div#info",
        "type": "SelectorText"
    }, {
        "id": "film_length",
        "multiple": false,
        "parentSelectors": ["detail_link"],
        "regex": "\\d+",
        "selector": "span[property='v:runtime']",
        "type": "SelectorText"
    }, {
        "id": "IMDb",
        "multiple": false,
        "parentSelectors": ["detail_link"],
        "regex": "(?<=[IMDb:\\s+])\\S*(?=\\d*$)",
        "selector": "div#info",
        "type": "SelectorText"
    }, {
        "id": "language",
        "multiple": false,
        "parentSelectors": ["detail_link"],
        "regex": "(?<=语言: )\\S+",
        "selector": "div#info",
        "type": "SelectorText"
    }, {
        "id": "alias",
        "multiple": false,
        "parentSelectors": ["detail_link"],
        "regex": "(?<=又名: )[\\u4e00-\\u9fa5A-Za-z0-9/()\\s]+(?=IMDb)",
        "selector": "div#info",
        "type": "SelectorText"
    }, {
        "id": "pic",
        "multiple": false,
        "parentSelectors": ["container"],
        "selector": "img",
        "type": "SelectorImage"
    }]
}

elasticsearch

{
    "mappings": {
      "properties": {
        "IMDb": {
          "type": "keyword",
          "copy_to": [
            "all"
          ]
        },
        "alias": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          },
          "copy_to": [
            "all"
          ],
          "analyzer": "ik_max_word",
          "search_analyzer": "ik_smart"
        },
        "all": {
          "type": "text",
          "analyzer": "ik_max_word",
          "search_analyzer": "ik_smart"
        },
        "area": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          },
          "copy_to": [
            "all"
          ],
          "analyzer": "ik_max_word",
          "search_analyzer": "ik_smart"
        },
        "director": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          },
          "copy_to": [
            "all"
          ],
          "analyzer": "ik_max_word",
          "search_analyzer": "ik_smart"
        },
        "film_length": {
          "type": "long"
        },
        "id": {
          "type": "keyword"
        },
        "language": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          },
          "copy_to": [
            "all"
          ],
          "analyzer": "ik_max_word",
          "search_analyzer": "ik_smart"
        },
        "link": {
          "type": "keyword"
        },
        "name": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          },
          "copy_to": [
            "all"
          ],
          "analyzer": "ik_max_word",
          "search_analyzer": "ik_smart"
        },
        "number": {
          "type": "long"
        },
        "photo": {
          "type": "keyword"
        },
        "review": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          },
          "copy_to": [
            "all"
          ],
          "analyzer": "ik_max_word",
          "search_analyzer": "ik_smart"
        },
        "score": {
          "type": "double"
        },
        "screenwriter": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          },
          "copy_to": [
            "all"
          ],
          "analyzer": "ik_max_word",
          "search_analyzer": "ik_smart"
        },
        "type": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          },
          "copy_to": [
            "all"
          ],
          "analyzer": "ik_max_word",
          "search_analyzer": "ik_smart"
        },
        "year": {
          "type": "long"
        }
      }
    }
  }

kibana

需要使用pipeline对索引字段进行处理，如对type 通过空格进行分割为数组等，可以参照官方文档或其他博客。

制作仪表板省略, 请自行搜索

SearchKit

参考 https://github.com/searchkit/searchkit-starter-app

【年度盛会】Elastic 中国开发者大会 2023，来自Elastic官方、极限科技、阿里、腾讯、谷歌、字节等业界专家带来的干货分享，我们诚挚邀请您共同参与。扫下图二维码或文末“阅读原文”即可报名购票参加（已报过名的参会者请勿重复报名），附上八折购票码（折扣码: 80OFF）。

大会议程

关于大会

Elastic 中国开发者大会 2023 是由 Elastic 官方、Elastic 中文社区和极限科技联合主办的开发者大会，作为中国国内唯一一个专门讨论 Elasticsearch 开源技术的大会，是中国最权威和最具实力干货的技术大会，其专业性和内容的质量一直以来在业内都是有口皆碑，大会最早发起于 2013 年初一个很小的线下聚会，之后每年迅速成长，往年大会的演讲嘉宾有来自 Elastic 官方、Google、腾讯、阿里巴巴、百度、字节、360、微博、美团、58、苏宁等众多公司的技术专家，带来过众多精彩的分享，与会听众大多为大数据领域相关的架构师、技术经理与一线开发工程师和运维工程师。

我们本着非盈利目的来举办大会，今年的大会将于2023年4月8日在深圳举行，举办开发者大会的目的是为中国广大的 Elasticsearch 开发者提供一个技术交流和学习切磋的地方，汇集业界众多的成功案例，集思广益，发散思维，促进社区和行业的进步。

大会时间：2023-04-08 08:00-18:00
大会地点：深圳深铁皇冠假日酒店
大会官网：https://conf.elasticsearch.cn