使用爬虫技术来抓取RabbitMQ数据

# luck-prometheus-exporter-mq # 介绍 使用Python爬虫技术来抓取RabbitMQ数据，对外提供接口，接口中提供抓取的RabbitMQ信息。 # 软件架构 ## 软件架构说明 - 项目名 - app - imgs 图片文件夹 - works Docker映射目录 - logs 日志文件夹 - LuckServer.log 日志文件 - LuckMq.db 数据库文件 - LuckMqConfig.json 配置文件 - LuckDbHelperUtils.py 数据库辅助类 - LuckMqApi.py MQ原生Api - LuckMqConstant.py 常量类 - LuckMqService.py 逻辑服务层 - LuckMqMain.py 启动入口 - requirements.txt 引用库管理文件 - LuckMqMain.service Linux服务制作文件 - Dockerfile # 构建镜像文件 - README.md # 说明文件 # 安装教程 ## 1 准备工作 - 调整配置文件内容 打开LuckMqConfig.json文件，调整下面的内容 ``` { "basic_auth1": "admin", # 认证鉴权账号 "basic_auth2": "luckserver", # 认证鉴权密码 "mq_server_port": 31902, # 服务启动端口号 "mq_name": "长春施工子系统", # 当前监控的MQ名字 "mq_host": "10.10.10.75", # 当前监控的MQ HOST "mq_port": 30673, # 当前监控的MQ 端口号 "mq_username": "guest", # 当前监控的MQ 登录账号 "mq_password": "luckserver" # 当前监控的MQ 登录密码 } ``` - 复制代码到服务器 示例目录：```/home/luckyun/luck-prometheus-exporter-mq``` ## 2 python3环境准备 ### 2.1 检测是否已安装python3 ```python3 --version``` python3安装成功，下面的步骤可忽略 ### 2.2 安装python3 - 安装Python的最新版本 ```sudo yum install python3``` - 安装完成之后，可以使用命令检测安装是否成功 ```python3 --version``` ### 2.3 设置python软链接 这里一般不需要设置，可以忽略。 >备注：如果需要设置python软链接，可按照下面的步骤操作 - 先删除默认的Python软链接： ```sudo rm -rf /usr/bin/python``` - 然后创建一个新的软链接指向需要的Python版本 ``` sudo ln -s /usr/bin/python3 /usr/bin/python sudo ln -s /usr/bin/pip3 /usr/bin/pip ``` - 如果想还原回原python2.7，不还原，下面的命令就不要执行 ``` sudo rm -rf /usr/bin/python sudo ln -s /usr/bin/ptyhon2.7 /usr/bin/python ``` ## 3 安装依赖 ### 3.1 使用命令安装依赖 ```pip3 install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple``` ### 3.2 安装不成功的，可以单独安装 psutil可能会安装不成功，这个时候不要着急，按照下面的步骤来操作 - 方案一 ``` 将psutil-5.9.6.tar.gz上传到对应的文件中 解压：tar zxvf psutil-5.9.6.tar.gz 进入目录：cd psutil-5.9.6 安装插件：yum -y install python-devel （如果提示缺少python.h头文件，执行此命令。） 安装psutil：python3 setup.py install ``` - 方案二（推荐此方案） - Ubuntu / Debian ``` sudo apt-get install gcc python3-dev pip install --no-binary :all: psutil ``` - RedHat / CentOS ``` sudo yum install gcc python3-devel pip install --no-binary :all: psutil ``` - Alpine ``` sudo apk add gcc python3-dev pip install --no-binary :all: psutil ``` ## 4 制作服务启动 - 复制service文件至system目录下 将服务文件`LuckMqMain.service`存放到下面的目录 `/etc/systemd/system` - 服务文件变动，需要重新加载systemd配置，没有就不执行 ```sudo systemctl daemon-reload``` - 启动服务 ```sudo systemctl start LuckMqMain.service``` - 检查服务的状态 ```sudo systemctl status LuckMqMain.service``` - 设置自动启动服务 ```sudo systemctl enable LuckMqMain.service``` - 停止服务 ```sudo systemctl stop LuckMqMain.service``` - 重启服务 ```sudo systemctl restart LuckMqMain.service``` ## 5 开放端口号 按照下面的操作步骤执行命令： ``` firewall-cmd --permanent --add-port=31902/tcp firewall-cmd --reload ``` # 使用说明 1. 服务启动成功 2. 浏览器输入 http://10.10.10.171:31902/, 验证服务部署状态 >注意：10.10.10.171要根据实际情况而定。 ``` { "msg": "luck-prometheus-exporter-mq install success", "url": ["/", "/metrics", "/readOverview", "/readVhosts", "/readQueues", "/createTable", "/deleteTable"] } ``` 3. 浏览器输入 http://10.10.10.171:31902/readOverview, 查看收集的MQ全部信息 ``` { "code": 1, "data": { "churn_rates": { "channel_closed": 86246, "channel_closed_details": { "rate": 0.0 }, "channel_created": 86440, "channel_created_details": { "rate": 0.0 }, "connection_closed": 685, "connection_closed_details": { "rate": 0.0 }, "connection_created": 632, "connection_created_details": { "rate": 0.0 }, "queue_created": 19, "queue_created_details": { "rate": 0.0 }, "queue_declared": 7067, "queue_declared_details": { "rate": 0.0 }, "queue_deleted": 17, "queue_deleted_details": { "rate": 0.0 } }, "cluster_name": "rabbit@my-rabbi", "contexts": [{ "cowboy_opts": "[{sendfile,false}]", "description": "RabbitMQ Management", "node": "rabbit@my-rabbi", "path": "/", "port": "15672", "ssl_opts": [] }], "disable_stats": false, "enable_queue_totals": false, "erlang_full_version": "Erlang/OTP 23 [erts-11.0.3] [source] [64-bit] [smp:8:8] [ds:8:8:10] [async-threads:128]", "erlang_version": "23.0.3", "exchange_types": [{ "description": "AMQP direct exchange, as per the AMQP specification", "enabled": true, "name": "direct" }, { "description": "AMQP fanout exchange, as per the AMQP specification", "enabled": true, "name": "fanout" }, { "description": "AMQP headers exchange, as per the AMQP specification", "enabled": true, "name": "headers" }, { "description": "AMQP topic exchange, as per the AMQP specification", "enabled": true, "name": "topic" }], "listeners": [{ "ip_address": "::", "node": "rabbit@my-rabbi", "port": 5672, "protocol": "amqp", "socket_opts": { "backlog": 128, "exit_on_close": false, "linger": [true, 0], "nodelay": true } }, { "ip_address": "::", "node": "rabbit@my-rabbi", "port": 25672, "protocol": "clustering", "socket_opts": [] }, { "ip_address": "::", "node": "rabbit@my-rabbi", "port": 15672, "protocol": "http", "socket_opts": { "cowboy_opts": { "sendfile": false }, "port": 15672 } }], "management_version": "3.8.5", "message_stats": { "ack": 7132202, "ack_details": { "rate": 0.0 }, "confirm": 6795249, "confirm_details": { "rate": 0.0 }, "deliver": 7145803, "deliver_details": { "rate": 0.0 }, "deliver_get": 7145918, "deliver_get_details": { "rate": 0.0 }, "deliver_no_ack": 0, "deliver_no_ack_details": { "rate": 0.0 }, "disk_reads": 572976, "disk_reads_details": { "rate": 0.0 }, "disk_writes": 9213927, "disk_writes_details": { "rate": 0.0 }, "drop_unroutable": 0, "drop_unroutable_details": { "rate": 0.0 }, "get": 115, "get_details": { "rate": 0.0 }, "get_empty": 0, "get_empty_details": { "rate": 0.0 }, "get_no_ack": 0, "get_no_ack_details": { "rate": 0.0 }, "publish": 7230535, "publish_details": { "rate": 0.0 }, "redeliver": 448889, "redeliver_details": { "rate": 0.0 }, "return_unroutable": 0, "return_unroutable_details": { "rate": 0.0 } }, "node": "rabbit@my-rabbi", "object_totals": { "channels": 194, "connections": 13, "consumers": 116, "exchanges": 173, "queues": 268 }, "product_name": "RabbitMQ", "product_version": "3.8.5", "queue_totals": { "messages": 5736561, "messages_details": { "rate": 0.0 }, "messages_ready": 5736560, "messages_ready_details": { "rate": 0.0 }, "messages_unacknowledged": 1, "messages_unacknowledged_details": { "rate": 0.0 } }, "rabbitmq_ver