329 lines
11 KiB
Python
329 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
"""
|
|
数据采集模块 - 从组态王API获取实时数据
|
|
"""
|
|
import json
|
|
import time
|
|
import ssl
|
|
import urllib.request
|
|
import urllib.error
|
|
from datetime import datetime
|
|
from database import get_connection
|
|
|
|
|
|
def get_unit_list():
|
|
"""获取所有工作面单位列表"""
|
|
conn = get_connection()
|
|
try:
|
|
with conn.cursor() as cursor:
|
|
cursor.execute("""
|
|
SELECT id, unit_code, unit_name, face_id, tag_prefix, enabled
|
|
FROM mine_units
|
|
WHERE enabled = 1
|
|
ORDER BY unit_code
|
|
""")
|
|
return cursor.fetchall()
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def get_unit_by_code(unit_code):
|
|
"""根据单位代码获取单位配置"""
|
|
conn = get_connection()
|
|
try:
|
|
with conn.cursor() as cursor:
|
|
cursor.execute("""
|
|
SELECT id, unit_code, unit_name, face_id, tag_prefix, enabled
|
|
FROM mine_units
|
|
WHERE unit_code = %s
|
|
""", (unit_code,))
|
|
return cursor.fetchone()
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def get_tag_list(unit_code, device_type=None, tag_category=None):
|
|
"""获取标签列表"""
|
|
conn = get_connection()
|
|
try:
|
|
# 验证单位是否存在
|
|
cursor = conn.cursor()
|
|
cursor.execute("SELECT id FROM mine_units WHERE unit_code = %s", (unit_code,))
|
|
result = cursor.fetchone()
|
|
if not result:
|
|
return []
|
|
|
|
conditions = []
|
|
params = []
|
|
|
|
if device_type:
|
|
# 设备类型映射
|
|
device_map = {
|
|
'shearer': 'Shearer',
|
|
'afc': 'AFC',
|
|
'loa': 'LOA',
|
|
'cru': 'CRU',
|
|
'emulsion': 'Emulsion',
|
|
'loop': 'Loop',
|
|
'spray': 'Spray',
|
|
}
|
|
if device_type in device_map:
|
|
conditions.append("tag_short_name LIKE %s")
|
|
params.append(f"{device_map[device_type]}%")
|
|
|
|
if tag_category:
|
|
conditions.append("tag_category = %s")
|
|
params.append(tag_category)
|
|
|
|
where_clause = ""
|
|
if conditions:
|
|
where_clause = f"WHERE {' AND '.join(conditions)}"
|
|
|
|
sql = f"SELECT tag_short_name, tag_category, unit as tag_unit FROM tag_definitions {where_clause} ORDER BY tag_short_name"
|
|
|
|
with conn.cursor() as cursor:
|
|
cursor.execute(sql, params)
|
|
return cursor.fetchall()
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def query_kingview_api(tag_names, api_url, user_handle):
|
|
"""调用组态王API获取实时数据"""
|
|
payload = {
|
|
"json_str": {
|
|
"user_handle": user_handle,
|
|
"tagNames": tag_names
|
|
},
|
|
"tagNames": tag_names,
|
|
"user_handle": user_handle
|
|
}
|
|
|
|
data = json.dumps(payload).encode('utf-8')
|
|
req = urllib.request.Request(
|
|
api_url,
|
|
data=data,
|
|
headers={
|
|
'Content-Type': 'application/json;charset=UTF-8',
|
|
'Accept': 'application/json, text/javascript, */*; q=0.01',
|
|
'X-Requested-With': 'XMLHttpRequest'
|
|
}
|
|
)
|
|
|
|
ctx = ssl.create_default_context()
|
|
ctx.check_hostname = False
|
|
ctx.verify_mode = ssl.CERT_NONE
|
|
|
|
try:
|
|
with urllib.request.urlopen(req, context=ctx, timeout=30) as response:
|
|
return json.loads(response.read().decode('utf-8'))
|
|
except urllib.error.URLError as e:
|
|
return {"error_code": -1, "message": str(e), "datas": []}
|
|
except Exception as e:
|
|
return {"error_code": -1, "message": str(e), "datas": []}
|
|
|
|
|
|
def parse_api_response(response):
|
|
"""解析API响应数据"""
|
|
results = []
|
|
if response.get('error_code') != 0:
|
|
return results
|
|
|
|
for item in response.get('datas', []):
|
|
for tag_val in item.get('TagValues', []):
|
|
if not tag_val:
|
|
continue
|
|
try:
|
|
tag_data = json.loads(tag_val) if isinstance(tag_val, str) else tag_val
|
|
results.append({
|
|
'tag_name': tag_data.get('N', ''),
|
|
'value': tag_data.get('V'),
|
|
'timestamp': tag_data.get('T'),
|
|
'quality': tag_data.get('Q', 0)
|
|
})
|
|
except (json.JSONDecodeError, KeyError):
|
|
continue
|
|
|
|
return results
|
|
|
|
|
|
def collect_data(unit_code, api_url, user_handle, batch_size=100, device_type=None, tag_category=None):
|
|
"""
|
|
采集指定单位的数据
|
|
返回: {'status': str, 'data': list, 'count': int, 'message': str}
|
|
"""
|
|
# 获取单位配置
|
|
unit = get_unit_by_code(unit_code)
|
|
if not unit:
|
|
return {'status': 'error', 'data': [], 'count': 0, 'message': f'未找到单位: {unit_code}'}
|
|
|
|
# 获取标签列表
|
|
tags = get_tag_list(unit_code, device_type=device_type, tag_category=tag_category)
|
|
if not tags:
|
|
return {'status': 'error', 'data': [], 'count': 0, 'message': '没有找到标签定义'}
|
|
|
|
# 构建完整标签名
|
|
prefix = unit['tag_prefix']
|
|
full_tag_names = [f"{prefix}{tag['tag_short_name']}" for tag in tags]
|
|
|
|
# 分批采集
|
|
all_data = []
|
|
total_batches = (len(full_tag_names) + batch_size - 1) // batch_size
|
|
|
|
for i in range(0, len(full_tag_names), batch_size):
|
|
batch = full_tag_names[i:i+batch_size]
|
|
batch_num = i // batch_size + 1
|
|
|
|
response = query_kingview_api(batch, api_url, user_handle)
|
|
data = parse_api_response(response)
|
|
all_data.extend(data)
|
|
|
|
# 批次间短暂延迟
|
|
if batch_num < total_batches:
|
|
time.sleep(0.1)
|
|
|
|
# 保存到数据库
|
|
saved = save_to_database(unit['id'], all_data, prefix)
|
|
|
|
return {
|
|
'status': 'success',
|
|
'unit_code': unit_code,
|
|
'unit_name': unit['unit_name'],
|
|
'tag_count': len(tags),
|
|
'data_count': len(all_data),
|
|
'saved': saved,
|
|
'message': f'采集完成，共{len(all_data)}条数据'
|
|
}
|
|
|
|
|
|
def save_to_database(unit_id, data, prefix):
|
|
"""保存采集数据到数据库"""
|
|
if not data:
|
|
return 0
|
|
|
|
conn = get_connection()
|
|
try:
|
|
with conn.cursor() as cursor:
|
|
# 创建快照
|
|
cursor.execute("""
|
|
INSERT INTO data_snapshots (unit_id, snapshot_time, tag_count, status)
|
|
VALUES (%s, NOW(), %s, 'success')
|
|
""", (unit_id, len(data)))
|
|
snapshot_id = cursor.lastrowid
|
|
|
|
# 批量插入实时数据
|
|
insert_sql = """
|
|
INSERT INTO data_realtime
|
|
(snapshot_id, unit_id, tag_short_name, tag_full_name, tag_value, tag_value_bool, quality, sample_time, category)
|
|
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
|
|
"""
|
|
|
|
values = []
|
|
for item in data:
|
|
full_name = item['tag_name']
|
|
# 提取短名称
|
|
short_name = full_name
|
|
if full_name.startswith(prefix):
|
|
short_name = full_name[len(prefix):]
|
|
|
|
val = item['value']
|
|
if val is None:
|
|
val_float = None
|
|
val_bool = None
|
|
elif isinstance(val, bool):
|
|
val_float = None
|
|
val_bool = 1 if val else 0
|
|
elif isinstance(val, (int, float)):
|
|
val_float = float(val)
|
|
val_bool = None
|
|
else:
|
|
try:
|
|
val_float = float(val)
|
|
val_bool = None
|
|
except (ValueError, TypeError):
|
|
val_float = None
|
|
val_bool = None
|
|
|
|
sample_time = item.get('timestamp') or datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
|
quality = item.get('quality', 0)
|
|
|
|
values.append((snapshot_id, unit_id, short_name, full_name, val_float, val_bool, quality, sample_time, ''))
|
|
|
|
# 批量插入
|
|
if values:
|
|
cursor.executemany(insert_sql, values)
|
|
|
|
conn.commit()
|
|
return len(values)
|
|
except Exception as e:
|
|
conn.rollback()
|
|
raise e
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def get_latest_snapshot(unit_code):
|
|
"""获取最新快照数据"""
|
|
conn = get_connection()
|
|
try:
|
|
with conn.cursor() as cursor:
|
|
# 获取unit_id
|
|
cursor.execute("SELECT id FROM mine_units WHERE unit_code = %s", (unit_code,))
|
|
unit = cursor.fetchone()
|
|
if not unit:
|
|
return None
|
|
|
|
# 获取最新快照
|
|
cursor.execute("""
|
|
SELECT * FROM data_snapshots
|
|
WHERE unit_id = %s
|
|
ORDER BY id DESC LIMIT 1
|
|
""", (unit['id'],))
|
|
snapshot = cursor.fetchone()
|
|
|
|
if snapshot:
|
|
# 获取快照数据
|
|
cursor.execute("""
|
|
SELECT dr.*, td.unit as tag_unit, td.tag_category as tag_category
|
|
FROM data_realtime dr
|
|
LEFT JOIN tag_definitions td ON dr.tag_short_name = td.tag_short_name
|
|
WHERE dr.snapshot_id = %s
|
|
ORDER BY dr.tag_short_name
|
|
""", (snapshot['id'],))
|
|
snapshot['data'] = cursor.fetchall()
|
|
|
|
return snapshot
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def get_history_data(unit_code, start_time, end_time, tag_name=None):
|
|
"""获取历史数据"""
|
|
conn = get_connection()
|
|
try:
|
|
with conn.cursor() as cursor:
|
|
cursor.execute("SELECT id FROM mine_units WHERE unit_code = %s", (unit_code,))
|
|
unit = cursor.fetchone()
|
|
if not unit:
|
|
return []
|
|
|
|
conditions = ["ds.unit_id = %s", "ds.snapshot_time BETWEEN %s AND %s"]
|
|
params = [unit['id'], start_time, end_time]
|
|
|
|
if tag_name:
|
|
conditions.append("dr.tag_short_name = %s")
|
|
params.append(tag_name)
|
|
|
|
sql = f"""
|
|
SELECT ds.snapshot_time, dr.tag_short_name, dr.tag_value, dr.tag_value_bool, dr.quality
|
|
FROM data_snapshots ds
|
|
JOIN data_realtime dr ON dr.snapshot_id = ds.id
|
|
WHERE {' AND '.join(conditions)}
|
|
ORDER BY ds.snapshot_time, dr.tag_short_name
|
|
"""
|
|
|
|
cursor.execute(sql, params)
|
|
return cursor.fetchall()
|
|
finally:
|
|
conn.close() |