linghtiin/DebugTool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

积累更新

Browse Source
This commit is contained in:
何 泽隆
2024-12-27 01:41:47 +08:00
parent 810bac464f
commit 00ddd6d68c
5 changed files with 1761 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

292
source/data_analysis.py
View File

@@ -14,32 +14,61 @@ from sqlalchemy import MetaData, Table, Column, String, Float, Integer
API_URL = "https://energy-iot.chinatowercom.cn/api/device/device/historyPerformance"
API_HEADER = {
"accept": "application/json, text/plain, */*",
"Accept-Encoding": "gzip, deflate, br, zstd",
"accept-language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6",
"Connection": "keep-alive",
"Content-Length": "170",
"Cookie": "HWWAFSESTIME=1732496660747; HWWAFSESID=880ee8eff3a2d23536; dc04ed2361044be8a9355f6efb378cf2=WyIzNTI0NjE3OTgzIl0",
"authorization": "Bearer eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOlsiIl0sInVzZXJfbmFtZSI6IndlYl9tYW5hZ2V8d2FuZ2xlaTQiLCJzY29wZSI6WyJhbGwiXSwiZXhwIjoxNzMyNTQ4Nzc5LCJ1c2VySWQiOjI0Mjg1LCJqdGkiOiI1Y2I3ZDA2ZC1hNzczLTQ5ZDQtYmRiMy05ZjdiOWNmMjY3ZTYiLCJjbGllbnRfaWQiOiJ3ZWJfbWFuYWdlIn0.Gpk9eYc3W4gcGLTG4aLSEL7E-VD-qPYsELcyyPuZ29e9STfa18WNcbosdIDulbH-5LDqY6i6aFAWpVm41bIgwzLZpAhlp2dthJcoBn9a90wWO4nz-Xi-MK9MVhNR0C28fyxv7h8jP_LRQkBi-pAzm2j65gkSTL3Vakky6znawZ5pY42U74lF-SnYdGyNFK3Ryy3xeMG4N1Pu_OJFdjXdGANUJZcXg1gh8WrERPo2SvuiCzOVTuUQZO7lRc8qGyILTKC-snz5CoonHaCWMaRzcV5VtCGd-yBncpyMQN9rGPqCiiLmuyTG29aZh5UfR_mloGCQHacqRmYSq3uGkPodBg",
"content-type": "application/json;charset=UTF-8",
"Cookie": "HWWAFSESID=455f2793ca86a3aaf0; HWWAFSESTIME=1734509454212; dc04ed2361044be8a9355f6efb378cf2=WyIyODM4MDM2MDcxIl0",
"authorization": "Bearer eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOlsiIl0sInVzZXJfbmFtZSI6IndlYl9tYW5hZ2V8d2FuZ2xlaTQiLCJzY29wZSI6WyJhbGwiXSwiZXhwIjoxNzM0NjYzNDQ5LCJ1c2VySWQiOjI0Mjg1LCJqdGkiOiJhZmZhNmY1My05ZDA4LTQ2ODUtODU3MS05YzA5ODAxMGJjZWYiLCJjbGllbnRfaWQiOiJ3ZWJfbWFuYWdlIn0.q0X4qrgL4wRUTZL8c_5oTIUGW0Lsivxw8pYQ1iMIqLnyJrUeS7IQKNRavMhc4NEdQ9uG6ZgFVHIj80HbzH8DHCxssCPLdv9_TXksI5podU2aU6Vjh6AaN1THFYAE2uflj1saBnQ5_gKiK0-cAcXDeJNSt_u6Cd9hI1ejEUPPzO_hLg-NLzch7yIB-HPvhoDNnl0n5pSYoQpT8XaKT14HezL3VQrLII69Vme38S2dMmmkiAeIyhHQi56kXZ11K45Lu5bHXv6fDg2Mfr9VgVuTleZldiO69BAmG0h1-HqTQuGE39jtGWrrCnFduRZR6VsaOWWJy3qyqUbXWMOli1Yy1g",
"Host": "energy-iot.chinatowercom.cn",
"Origin": "https://energy-iot.chinatowercom.cn",
"Sec-Fetch-Dest": "empty",
"Sec-Fetch-Mode": "cors",
"Sec-Fetch-Site": "same-origin",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
"accept": "application/json, text/plain, */*",
"accept-language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6",
"content-type": "application/json;charset=UTF-8",
"sec-ch-ua": "\"Microsoft Edge\";v=\"131\", \"Chromium\";v=\"131\", \"Not_A Brand\";v=\"24\"",
"sec-ch-ua-mobile": "?0",
"sec-ch-ua-platform": "Windows",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
}
SemaMap = {
'apt_temp': ('0305117001', 'adapter', "设备温度"),
'apt_volt_in': ('0305118001', 'adapter', "输入电压"),
'apt_curr_in': ('0305119001', 'adapter', "输入电流"),
'apt_volt_out': ('0305120001', 'adapter', "输出电压"),
'apt_curr_out': ('0305121001', 'adapter', "输出电流"),
'apt_power_out': ('0305122001', 'adapter', "输出功率"),
SemaMap_adapter = {
'facturer': ('0305113001', 'adapter', False, "厂家"),
'version': ('0305114001', 'adapter', False, "软件版本"),
'model': ('0305115001', 'adapter', False, "型号"),
'status': ('0305116001', 'adapter', False, "开关机状态"),
'temp': ('0305117001', 'adapter', True, "温度"),
'volt_in': ('0305118001', 'adapter', True, "输入电压"),
'curr_in': ('0305119001', 'adapter', True, "输入电流"),
'volt_out': ('0305120001', 'adapter', True, "输出电压"),
'curr_out': ('0305121001', 'adapter', True, "输出电流"),
'power_out': ('0305122001', 'adapter', True, "输出功率"),
}
SemaMap_meter = {
'mtr_id': ('0305123001', 'meter', False, "电表号"),
'mtr_volt': ('0305124001', 'meter', True, "直流电压"),
'mtr_curr': ('0436101001', 'meter', True, "直流总电流"),
'mtr_power': ('0436102001', 'meter', True, "总有功功率"),
'mtr_energy_total': ('0305125001', 'meter', True, "总有功电能"),
'mtr_energy_daily': ('0305126001', 'meter', True, "日有功电能"),
'mtr_energy_total_T': ('0305127001', 'meter', True, "尖时段总正向有功电能"),
'mtr_energy_total_P': ('0305128001', 'meter', True, "峰时段总正向有功电能"),
'mtr_energy_total_F': ('0305129001', 'meter', True, "平时段总正向有功电能"),
'mtr_energy_total_V': ('0305130001', 'meter', True, "谷时段总正向有功电能"),
'mtr_energy_daily_T': ('0305131001', 'meter', True, "尖时段日正向有功电能"),
'mtr_energy_daily_P': ('0305132001', 'meter', True, "峰时段日正向有功电能"),
'mtr_energy_daily_F': ('0305133001', 'meter', True, "平时段日正向有功电能"),
'mtr_energy_daily_V': ('0305134001', 'meter', True, "谷时段日正向有功电能"),
}
API_Map = {
'search_dev': ['https://energy-iot.chinatowercom.cn/api/device/device/page', None],
'dev_info': ['https://energy-iot.chinatowercom.cn/api/device/device/devInfo', None],
'perf_real': ['https://energy-iot.chinatowercom.cn/api/device/device/perfReal', None],
'history': ['https://energy-iot.chinatowercom.cn/api/device/device/historyPerformance', [SemaMap_adapter, SemaMap_meter]],
'page': ['https://energy-iot.chinatowercom.cn/api/device/device/page', None],
'station': ['https://energy-iot.chinatowercom.cn/api/device/station/detail/', None],
}
class Lamina_Data(object):
@@ -60,6 +89,16 @@ class Lamina_Data(object):
)
metadata.create_all(self.engine)
if 'log' not in metadata.tables:
log_table = Table(
'log', metadata,
Column('dev', String(50)),
Column('mid', String(50)),
Column('Timestamp_start', Integer),
Column('Timestamp_end', Integer),
)
metadata.create_all(self.engine)
self.data = {
'history': pd.read_sql_table('history', self.engine),
}
@@ -75,33 +114,34 @@ class Lamina_Data(object):
data_memory = self.data['history']
data_file = pd.read_sql_table('history', self.engine)
# 合并新数据和现有数据
combined_df = pd.concat([data_file, data_memory], ignore_index=True)
# 通过 'dev', 'mid', 'time' 三列进行去重
combined_df.drop_duplicates(subset=['dev', 'mid', 'time'], inplace=True)
# 将去重后的数据插入数据库
combined_df.to_sql('history', self.engine, if_exists='replace', index=False)
merged_df = pd.merge(data_memory, data_file[['dev', 'mid', 'time']], on=['dev', 'mid', 'time'], how='left', indicator=True)
filtered_data_memory = merged_df[merged_df['_merge'] == 'left_only'].drop(columns='_merge')
filtered_data_memory.to_sql('history', self.engine, if_exists='append', index=False)
print(f"成功插入 {len(combined_df)} 条数据")
print(f"成功插入 {len(filtered_data_memory)} 条数据")
return len(combined_df)
return len(filtered_data_memory)
def get_histoty_data(self, device_id, data_type, time_start:int, time_end:int):
def save_data(func):
""" 保存函数返回数据 """
def wrapper(*args, **kwds):
self: Lamina_Data = args[0]
result = func(*args, **kwds)
if isinstance(result, pd.DataFrame):
self.data['history'] = pd.concat([self.data['history'], result], ignore_index=True)
self.save_history_data()
return result
return wrapper
def get_histoty_data_by_database(self, device_id, data_type, time_start:int, time_end:int):
""" 读取历史数据 """
flag = 0
time_Interval = 20 * 60
database = self.data['history']
filter_data = database[database['dev'] == device_id &
database['mid'] == SemaMap[data_type][0] &
database['mid'] == SemaMap_adapter[data_type][0] &
database['time'].between(time_start, time_end)]
if filter_data.shape[1] == 0 or (filter_data['time'].min() - time_start > time_Interval) or (time_end - filter_data['time'].max() > time_Interval):
""" 需要从网页获取数据 """
flag = 1
return filter_data
@save_data
def get_history_data_by_net(self, device_id, data_type, time_start:int, time_end:int, header=None):
""" 读取信号量历史数据, 返回接口json数据 """
if header is None:
@@ -122,38 +162,163 @@ class Lamina_Data(object):
json_data = req.json()
if json_data['code'] == 200:
""" 数据读取成功 """
print(f"Get data success, len={len(json_data['data'])}")
print(f"Get data success, mid={data_type[0]}, len={len(json_data['data'])}")
table_data = pd.DataFrame(json_data['data'], columns=['collectTime', 'mid', 'midName', 'value'])
table_data['time'] = table_data['collectTime'].apply(lambda x: int(time.mktime(time.strptime(x, r"%Y-%m-%d %H:%M:%S"))))
table_data['dev'] = device_id
return table_data[['dev', 'mid', 'time', 'value']]
else:
print(f"Get data fail, code={json_data['code']}, msg=\n\t{json_data['msg']}")
print(f"Get data fail, code={json_data['code']}, msg=\n\t{json_data['message']}")
return pd.DataFrame([], columns=['dev', 'mid', 'time', 'value'])
def get_real_data_by_net(self, device_id, fsu_id=None, header=None):
""" 读取设备当前数据, 返回接口json数据 """
if header is None:
header = self.api_origin['header']
body = {
"businessType": "7",
"devType": device_id[3:7],
"deviceCodes": device_id,
"type": "遥测"
}
if device_id[3:7] != "0101":
if fsu_id is None:
raise ValueError(f"Missing required parameters: fsu_id={fsu_id}")
body["fsuCode"] = fsu_id
req = requests.post(API_Map['perf_real'][0], json=body, headers=header)
json_data = req.json()
if json_data['code'] == 200:
""" 数据读取成功 """
print(f"Get data success, len={len(json_data['data'])}")
table_data = pd.DataFrame(json_data['data'])
column_name = sorted(table_data.columns)
table_data['time'] = table_data['updateTime'].apply(lambda x: int(time.mktime(time.strptime(x, r"%Y-%m-%d %H:%M:%S"))))
table_data = table_data[['time', *column_name]].drop(columns='updateTime')
return table_data
else:
print(f"Get data fail, code={json_data['code']}, msg=\n\t{json_data['message']}")
return pd.DataFrame([])
def get_devinfo_data_by_net(self, device_id, data_type, time_start:int, time_end:int, header=None):
""" 读取设备信息, 返回接口json数据 """
if header is None:
header = self.api_origin['header']
body = {
"businessType": "7",
"id": int(data_type),
}
req = requests.post(API_Map['dev_info'][0], json=body, headers=header)
json_data = req.json()
if json_data['code'] == 200:
""" 数据读取成功 """
print(f"Get data success, len={len(json_data['data'])}")
table_data = pd.DataFrame(json_data['data'])
return table_data
else:
print(f"Get data fail, code={json_data['code']}, msg=\n\t{json_data['message']}")
return pd.DataFrame([], columns=['dev', 'mid', 'time', 'value'])
def spider_adapter(self, device_id:str, time_start:int, time_end:int):
""" 爬取适配器数据 """
result = {}
columns_adapter = list(filter(lambda x: SemaMap_adapter[x][2], SemaMap_adapter.keys()))
data_adapter = pd.DataFrame([], columns=['time', *columns_adapter])
for k in columns_adapter:
result[k] = self.get_history_data_by_net(device_id, SemaMap_adapter[k], time_start, time_end)
if data_adapter.empty:
data_adapter.time = result[k].time
data_adapter[k] = result[k].value.apply(float)
return data_adapter
def spider_meter(self, device_id:str, time_start:int, time_end:int):
""" 爬取电表数据 """
result = {}
columns_meter = list(map(lambda x: x[4:], filter(lambda x: SemaMap_meter[x][2], SemaMap_meter.keys())))
data_meter = pd.DataFrame([], columns=['time', *columns_meter])
for k, v in SemaMap_meter.items():
if v[2]:
result[k] = self.get_history_data_by_net(device_id, v, time_start, time_end)
if data_meter.empty:
data_meter.time = result[k].time
data_meter[k[4:]] = result[k].value.apply(float)
return data_meter
def spider_station(self, device_id:str, time_start:int, time_end:int, header=None):
""" 爬取站点数据 """
if header is None:
header = self.api_origin['header']
body = {
"devType": "",
"accessPointId": "",
"pageNum": 1,
"pageSize": 10,
"businessType": "7",
"devCode": device_id,
"deptIds": []
}
req = requests.post(API_Map['search_dev'][0], json=body, headers=header)
json_data = req.json()
if json_data['code'] != 200:
""" 数据读取失败 """
print(f"Get data fail, code={json_data['code']}, msg=\n\t{json_data['message']}")
return pd.DataFrame([])
elif search_dev := json_data['rows']:
print(f"Search device success, len={len(search_dev)}")
station_id = search_dev[0]['stationCode']
else:
print(f"Search device fail.")
return pd.DataFrame([])
body = {
"businessType": "7",
"stationCode": station_id,
}
time.sleep(0.5)
print(f"Get Data for Station: {station_id}")
req = requests.post(API_Map['page'][0], json=body, headers=header)
json_data = req.json()
for dev in sorted(json_data['rows'], key=lambda x: x['devCode']):
print(f"Dev: {dev['devTypeName']}, id={dev['devCode']}")
time.sleep(0.5)
fsu_id = dev['parentCode'] if 'parentCode' in dev.keys() else None
self.get_real_data_by_net(dev['devCode'], fsu_id, header=header)
time.sleep(0.5)
match dev['devType']:
case "0101":
fsu_id = dev['devCode']
case "0102":
self.spider_adapter(dev['devCode'], time_start, time_end)
case "0103":
self.spider_meter(dev['devCode'], time_start, time_end)
print(f"Station Done.")
def spider(self, device_id:str, time_start:int, time_end:int):
""" 通用爬虫 """
if device_id[:8] == "TTE0102DX":
""" 适配器数据 """
self.spider_adapter(device_id, time_start, time_end)
elif device_id[:8] == "TTE0103DX":
""" 电表数据 """
self.spider_meter(device_id, time_start, time_end)
def graphs_adapter(self, device_id, time_start:int|str, time_end:int|str):
""" 绘制图表-适配器数据 """
if type(time_start) is str:
time_start = time.mktime(time.strptime(time_start, r"%Y-%m-%d %H:%M:%S"))
if type(time_end) is str:
time_end = time.mktime(time.strptime(time_end, r"%Y-%m-%d %H:%M:%S"))
data_volt_in = self.get_history_data_by_net(device_id, SemaMap["apt_volt_in"], time_start, time_end)
data_curr_in = self.get_history_data_by_net(device_id, SemaMap["apt_curr_in"], time_start, time_end)
data_volt_out = self.get_history_data_by_net(device_id, SemaMap["apt_volt_out"], time_start, time_end)
data_power_out = self.get_history_data_by_net(device_id, SemaMap["apt_power_out"], time_start, time_end)
data = self.spider_adapter(device_id, time_start, time_end)
combined_df = pd.concat([data_volt_in, data_curr_in, data_volt_out, data_power_out], ignore_index=True)
self.data['history'] = pd.concat([self.data['history'], combined_df], ignore_index=True)
self.save_history_data()
self.chart_adapter(data)
data_adapter = pd.DataFrame([], columns=['time', 'volt_in', 'volt_out', 'curr_in', 'power_out'])
data_adapter.time = data_volt_in.time
data_adapter.volt_in = data_volt_in.value
data_adapter.volt_out = data_volt_out.value
data_adapter.curr_in = data_curr_in.value
data_adapter.power_out = data_power_out.value
return data_adapter
return data
def chart_adapter(self, data_adapter):
""" 绘制适配器信息图表 """
@@ -204,15 +369,30 @@ def sim_data_apt(times:tuple[int, int]):
if __name__=='__main__':
# API_HEADER['authorization'] = 'Bearer eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOlsiIl0sInVzZXJfbmFtZSI6IndlYl9tYW5hZ2V8d2FuZ2xlaTQiLCJzY29wZSI6WyJhbGwiXSwiZXhwIjoxNzMyNTQzMzIyLCJ1c2VySWQiOjI0Mjg1LCJqdGkiOiI2MGZlZjYxNC0zM2QyLTQ5NjMtYjM5Mi1mNTJlYzJhM2RmYTEiLCJjbGllbnRfaWQiOiJ3ZWJfbWFuYWdlIn0.jumI9T8swSPCKITrCU06DWRea0tpvwCidw6jFP-F26JH0fRCp801fbzPs5kHkrgh-sm_ayzO2mD9NCkC2hoC-YDQs8HCvXHZXwtYpn3qJ6vpWRQrMr8Vwk8HnuxbFUn-8Ccs88hHOa8rtrEVsglfl0gqU1C8LL1rsHxgYI3MAwwU_eHo74jNjSHuIn79iq4Gyxc6lial6Pb6WWOw1erqKRGRQPcfUtAXMjwXbQ03vFk6G-As_u353yKvznsGR4A70hFHkJJGugK3en8kJO3_g3sE6ddcx0CYicucRLckro4Md_S8mIPkw2RgGEK38vvEo2fqiyT8E67dC5D4_zMw5A'
data_lamina = Lamina_Data('sqlite:///result/chinatowercom.db')
""" 主体调用流程 """
if hasattr(__builtins__,"__IPYTHON__"):
path_db = '../result/chinatowercom.db'
else:
path_db = 'result/chinatowercom.db'
API_HEADER['Cookie'] = "HWWAFSESTIME=1735108780906; HWWAFSESID=1c91597e07b0014c4d; dc04ed2361044be8a9355f6efb378cf2=WyIzNTI0NjE3OTgzIl0"
API_HEADER['authorization'] = 'Bearer eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOlsiIl0sInVzZXJfbmFtZSI6IndlYl9tYW5hZ2V8d2FuZ2xlaTQiLCJzY29wZSI6WyJhbGwiXSwiZXhwIjoxNzM1MjI2NDAyLCJ1c2VySWQiOjI0Mjg1LCJqdGkiOiJmNzA1ZTlkZC1mNTA5LTQwYzUtODFhNi0zYzdlMzhhZjE0ODgiLCJjbGllbnRfaWQiOiJ3ZWJfbWFuYWdlIn0.e8p-hKWoFyLJbtqKEvlIbLpz-_OB7Ak32d8qdTHNZEny12lUrUE0YYrWQTu0gGtT-eRNDJ62q51IUYOkM_5Ou0Qk2HLouR9-iygtgtjIno72466bv_ao5wvD2PZihXKaKet_c9mnpOqDpvaaApAU4_rk0u6Pg7uJG4stV-akaaMMqRLR-cK5ARePeyHophyGUx80kkSlnhYfGP2rJjEFva36iPaCzM6oiezObMoXWtAPw67vPS-5saTWnjYLrzxr3_s5Idk1pwWPNWfSa6Rl_YMKKiTdtWAEepyrxxOWVfMaeAQYt-ndHhxyBPjRluDTSwUViWmDidoFxkKPMQixVw'
data_lamina = Lamina_Data('sqlite:///' + path_db)
today = datetime.datetime.today()
yesterday = today - datetime.timedelta(days=1)
today_midnight = today.replace(hour=0, minute=0, second=0, microsecond=0)
yesterday_midnight = yesterday.replace(hour=0, minute=0, second=0, microsecond=0)
today_midnight_timestamp = time.mktime(today_midnight.timetuple())
yesterday_midnight_timestamp = time.mktime(yesterday_midnight.timetuple())
# 依据站点内设备爬取整个站点的实时与历史数据
data = data_lamina.spider_station('TTE0102DX2406272727', yesterday_midnight_timestamp, today_midnight_timestamp)
# data = sim_data_apt(('2024-10-1 00:00:00', '2024-10-1 12:00:00'))
# chart_apt(data)
table_apt = data_lamina.graphs_adapter('TTE0102DX2406180988', '2024-11-23 00:00:00', '2024-11-26 00:00:00')
data_lamina.chart_adapter(table_apt)
while True:
plt.waitforbuttonpress()
if not hasattr(__builtins__,"__IPYTHON__"):
table_apt = data_lamina.graphs_adapter('TTE0102DX2406180988', '2024-11-23 00:00:00', '2024-11-26 00:00:00')
while True:
plt.waitforbuttonpress()