MySQL --> Canal-Server --> MQ --> Canal-adapter --> ES
$ docker compose up -d$ mysql -uroot -p
-- 创建同步账号
CREATE USER canal IDENTIFIED BY 'canal';
GRANT SELECT, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'canal'@'%';
-- GRANT ALL PRIVILEGES ON *.* TO 'canal'@'%';
FLUSH PRIVILEGES;
-- 创建测试数据
CREATE DATABASE testdb;
USE testdb;
CREATE TABLE `article` (
`id` bigint NOT NULL AUTO_INCREMENT,
`title` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL,
`body` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL,
`last_time` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
PRIMARY KEY (`id`) USING BTREE
);
-- INSERT INTO `article`(`title`, `body`) VALUES ('标题', '正文内容');配置已经包含在 docker-compose
验证：
# 日志路径 canal-server/logs/{$destination}/{$destination}.log
$ docker exec -it mysql-canal-es-canal-server-1 tail -f canal-server/logs/test/test.log
2023-01-05 16:52:48.915 [destination = test , address = mysql/172.25.0.3:3306 , EventParser] WARN c.a.o.c.p.inbound.mysql.rds.RdsBinlogEventParserProxy - ---> begin to find start position, it will be long time for reset or first position
2023-01-05 16:52:48.923 [destination = test , address = mysql/172.25.0.3:3306 , EventParser] WARN c.a.o.c.p.inbound.mysql.rds.RdsBinlogEventParserProxy - prepare to find start position just show master status
2023-01-05 16:52:51.450 [destination = test , address = mysql/172.25.0.3:3306 , EventParser] WARN c.a.o.c.p.inbound.mysql.rds.RdsBinlogEventParserProxy - ---> find start position successfully, EntryPosition[included=false,journalName=mysql-bin.000003,position=4,serverId=1,gtid=<null>,timestamp=1672908641000] cost : 2508ms , the next step is binlog dump日志中可见 find start position successfully 即可
配置已经包含在 docker-compose 中
- 新建 topic
验证：
$ docker exec -it mysql-canal-es-kafka-1 sh -c '/opt/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --list'
my-topic可见 my-topic 即可
在 Kibana 的 Dev Tools 中创建
http://localhost:5601/app/dev_tools#/console# 建立索引
PUT /blog_article
{
"settings": {
"number_of_shards": "1",
"number_of_replicas": "0"
},
"mappings": {
"properties": {
"title":{
"type": "text"
},
"body":{
"type": "text"
},
"last_time":{
"type": "date"
}
}
}
}验证：
# 验证索引
GET /_cat/indices
green open blog_article cOgb1VOmT2aFNmclLaQAhQ 1 0 0 0 226b 226b
# 多字段查询，默认情况下 汉字会被拆分每个字匹配，上结巴 分词插件
GET /blog_article/_search
{
"query": {
"multi_match": {
"query": "关键词",
"fields": ["title", "body"]
}
}
}可见索引 blog_article 状态且为 green 即可
更多 es 的操作
# URL 查询
http://localhost:9200/blog_article/_search?q=title:哈哈
# 查询
GET /blog_article/_search
{
"query": {
"match": {
"title": "哈哈"
}
},
"_source": ["title", "body"],
"highlight": {
"fields": {
"title": {}
}
}
}
# 多字段查询
GET /blog_article/_search
{
"query": {
"multi_match": {
"query": "哈哈",
"fields": ["title", "body"]
}
}
}
# 查看具体文档
GET /blog_article/_doc/1/
POST /blog_article/_open
GET /blog_article/_mapping
# 查看所有索引
GET /_cat/indices
# 删除索引
DELETE blog_article使用 canal-adapter 跳过这步
$ export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
$ python canal-client.py官方：canal-adapter 替代 canal-client.py https://github.com/alibaba/canal/releases/download/canal-1.1.6/canal.adapter-1.1.6.tar.gz
- conf/application.yml
- conf/es7/blog_article.yml
$ cd canal.adapter-1.1.6
$ bin/startup.sh
$ tail -f logs/adapter/adapter.log插入、更新数据测试
INSERT INTO `article`(`title`, `body`) VALUES ('我是标题', '我是正文内容');
UPDATE `article` SET `title`='标题已改变' WHERE `id`=1;# 多字段查询
GET /blog_article/_search
{
"query": {
"multi_match": {
"query": "哈哈",
"fields": ["title", "body"]
}
}
}如果未在es中查找到结果，也许您需要手动ETL
# 手动 ETL
$ curl http://127.0.0.1:8081/etl/es7/blog_article.yml -X POST
{"succeeded":true,"resultMessage":"导入ES 数据：1 条"}%
$ curl http://127.0.0.1:8081/destinations
$ curl http://127.0.0.1:8081/syncSwitch/example/off -X PUT
$ curl http://127.0.0.1:8081/syncSwitch/test/on -X PUT
GET article/_search